diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..b1f35fc4f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.js linguist-language=java +*.css linguist-language=java +*.html linguist-language=java +*.vue linguist-language=java \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..ea62ffcc0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,106 @@ +name: Bug report +title: "[Bug] " +description: Problems and issues with code of DSS +labels: [bug, triage] +body: + - type: markdown + attributes: + value: | + Thank you for reporting the problem! + Please make sure what you are reporting is a bug with reproducible steps. To ask questions + or share ideas, pleae post on our [Discussion page](https://github.com/WeBankFinTech/DataSphereStudio/discussions) instead. + + - type: checkboxes + attributes: + label: Search before asking + description: > + Please make sure to search in the [issues](https://github.com/WeBankFinTech/DataSphereStudio/issues) first to see + whether the same issue was reported already. + options: + - label: > + I searched the [issues](https://github.com/WeBankFinTech/DataSphereStudio/issues) and found no similar + issues. + required: true + + - type: dropdown + attributes: + label: DSS Component + description: | + What component are you using? DSS has many modules, please make sure to choose the module that + you found the bug. + multiple: true + options: + - "dss-commons" + - "dss-appconn" + - "dss-framework" + - "dss-orchestrator" + - "dss-standard" + - "dss-plugins" + - "dss-apps/dss-apiservice" + - "dss-web/dss-scriptis" + - "dss-web/dss-workflow" + - "dss-web/workspace" + - "dss-web/dss-apiservice" + - "dss-web/framework" + validations: + required: true + + - type: textarea + attributes: + label: What happened + What you expected to happen + description: Describe 1. the bug 2. expected behavior 3. useful information (e.g., logs) + placeholder: > + Please provide the context in which the problem occurred and explain what happened. Further, + To Reproduce Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '.... 4. See error + please also explain why you think the behaviour is erroneous. It is extremely helpful if you can + copy and paste the fragment of logs showing the exact error messages or wrong behaviour here. + + **NOTE**: Expected behavior A clear and concise description of what you expected to happen.Screenshots If applicable, add screenshots to help explain your problem. + validations: + required: true + + - type: textarea + attributes: + label: Relevent platform + description: The platform where you occurred this issue + placeholder: > + Please specify Desktop or Smartphone, Version / Dependencies / OS / Browser + validations: + required: true + + - type: textarea + attributes: + label: Reproduction script + description: > + Please provide a reproducible script. Providing a narrow reproduction (minimal / no external dependencies) will + help us triage and address issues in the timely manner! + placeholder: > + Please provide a short code snippet (less than 50 lines if possible) that can be copy-pasted to + reproduce the issue. The snippet should have **no external library dependencies** + (i.e., use fake or mock data / environments). + + **NOTE**: If the code snippet cannot be run by itself, the issue will be marked as "needs-repro-script" + until the repro instruction is updated. + validations: + required: true + + - type: textarea + attributes: + label: Anything else + description: Anything else we need to know? + placeholder: > + How often does this problem occur? (Once? Every time? Only when certain conditions are met?) + Any relevant logs to include? Are there other relevant issues? + + - type: checkboxes + attributes: + label: Are you willing to submit a PR? + description: > + This is absolutely not required, but we are happy to guide you in the contribution process + especially if you already have a good understanding of how to implement the fix. + options: + - label: Yes I am willing to submit a PR! + + - type: markdown + attributes: + value: "Thanks for completing our form!" diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..c66904579 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: fasle +contact_links: + - name: Ask a question or get support + url: https://github.com/WeBankFinTech/DataSphereStudio/discussions + about: Ask a question or request support for using DSS \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..056c57005 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,63 @@ +name: DSS feature request +description: Suggest an idea for DSS project +title: "[Feature] " +labels: [enhancement] +body: + - type: markdown + attributes: + value: | + Thank you for finding the time to propose a new feature! + We really appreciate the community efforts to improve DSS. + - type: checkboxes + attributes: + label: Search before asking + description: > + Please make sure to search in the [issues](https://github.com/WeBankFinTech/DataSphereStudio/issues) first to see + whether the same feature was requested already. + options: + - label: > + I had searched in the [issues](https://github.com/WeBankFinTech/DataSphereStudio/issues) and found no similar + feature requirement. + required: true + - type: textarea + attributes: + label: Problem Description + description: Is your feature request related to a problem? Please describe. + + - type: textarea + attributes: + label: Description + description: A short description of your feature + + - type: textarea + attributes: + label: Use case + description: > + Describe the use case of your feature request. + placeholder: > + Describe the solution you'd like A clear and concise description of what you want to happen. + + - type: textarea + attributes: + label: solutions + description: Describe alternatives you've considered A clear and concise description of any alternative solutions or features you've considered. + + - type: textarea + attributes: + label: Anything else + description: Anything else we need to know? + placeholder: > + Additional context Add any other context or screenshots about the feature request here. + + - type: checkboxes + attributes: + label: Are you willing to submit a PR? + description: > + This is absolutely not required, but we are happy to guide you in the contribution process + especially if you already have a good understanding of how to implement the feature. + options: + - label: Yes I am willing to submit a PR! + + - type: markdown + attributes: + value: "Thanks for completing our form!" diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..f705afc3e --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,28 @@ +### What is the purpose of the change +(For example: AppConn Core defines the abstractions and interfaces of the AppConn core functions. +It is the core concept that enables DSS to easily and quickly integrate various upper-layer web systems. +Related issues: #590. ) + +### Brief change log +(for example:) +- Define the core abstraction and interfaces of the AppConn; +- Define the core abstraction and interfaces of the three-level specifications of AppConn. + +### Verifying this change +(Please pick either of the following options) +This change is a trivial rework / code cleanup without any test coverage. +(or) +This change is already covered by existing tests, such as (please describe tests). +(or) +This change added tests and can be verified as follows: +(example:) +- Added tests for creating and execute the workflow contains the nodes that DSS has integrated to go through and verify the availability of different AppConns. + +### Does this pull request potentially affect one of the following parts: +- Dependencies (does it add or upgrade a dependency): (yes / no) +- Anything that affects deployment: (yes / no / don't know) +- The Core framework, i.e., AppConn, Orchestrator, ApiService.: (yes / no) + +### Documentation +- Does this pull request introduce a new feature? (yes / no) +- If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..bf9577849 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,56 @@ +# +# Copyright 2019 WeBank. +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: DataSphereStudio CI Actions + +on: + push: + pull_request: + +jobs: + build: + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [14.17.3] + # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ + + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: 8 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node-version }} + - name: Build backend by maven + run: | + mvn -N install + mvn clean package + - name: Build frontend by node.js + run: | + cd web + npm install lerna -g + lerna bootstrap + npm run build diff --git a/.github/workflows/check_license.yml b/.github/workflows/check_license.yml new file mode 100644 index 000000000..51961051d --- /dev/null +++ b/.github/workflows/check_license.yml @@ -0,0 +1,48 @@ +# +# Copyright 2019 WeBank. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: DataSphereStudio License check + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'adopt' + - name: mvn -N install + run: + mvn -N install + - name: License check with Maven + run: | + rat_file=`mvn apache-rat:check | { grep -oe "\\S\\+/rat.txt" || true; }` + echo "rat_file=$rat_file" + if [[ -n "$rat_file" ]];then echo "check error!" && cat $rat_file && exit 123;else echo "check success!" ;fi + - name: Upload the report + uses: actions/upload-artifact@v2 + with: + name: license-check-report + path: "**/target/rat.txt" diff --git a/.github/workflows/dead-link-checker.yml b/.github/workflows/dead-link-checker.yml new file mode 100644 index 000000000..2844f0965 --- /dev/null +++ b/.github/workflows/dead-link-checker.yml @@ -0,0 +1,20 @@ +name: Link checker + +on: + pull_request: + push: + schedule: + - cron: "0 5 * * *" + +jobs: + CheckDeadLinks: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'no' + use-verbose-mode: 'yes' + folder-path: '../' + config-file: '.github/workflows/dlc.json' \ No newline at end of file diff --git a/.github/workflows/dlc.json b/.github/workflows/dlc.json new file mode 100644 index 000000000..ca6020cf0 --- /dev/null +++ b/.github/workflows/dlc.json @@ -0,0 +1,17 @@ +{ + "ignorePatterns": [ + { + "pattern": "https://github.com/WeBankFinTech/DataSphereStudio-Doc" + } + ], + "timeout": "10s", + "retryOn429": true, + "retryCount": 10, + "fallbackRetryDelay": "1000s", + "aliveStatusCodes": [ + 200, + 401, + 403 + ] +} + diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..7bc2f236d --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +.DS_Store +.cache + +# for ide +*.iml +*.ipr +*.iws +*.pyc +*.pyo +*.swp +.idea/ +.idea_modules/ +.project +.pydevproject +.scala_dependencies +.settings +.classpath + +# For SBT +.jvmopts + +# For Node.js +node_modules/ + +# generated file +.mvn/wrapper/maven-wrapper.jar +dist/ +out/ +target/ + +# log folder +logs/ +*.log \ No newline at end of file diff --git a/README-ZH.md b/README-ZH.md index ccf415236..5a2eb02d2 100644 --- a/README-ZH.md +++ b/README-ZH.md @@ -1,4 +1,4 @@ -![DSS](images/en_US/readme/DSS.png) +![DSS](images/en_US/readme/DSS_logo.png) ==== [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) @@ -7,157 +7,179 @@ ## 引言 -DataSphere Studio(简称DSS)是微众银行大数据平台——WeDataSphere,自研的一站式数据应用开发管理门户。 +        DataSphere Studio(简称 DSS)是微众银行自研的数据应用开发管理集成框架。 -基于 [**Linkis**](https://github.com/WeBankFinTech/Linkis) 计算中间件构建,可轻松整合上层各数据应用系统,让数据应用开发变得简洁又易用。 +        基于插拔式的集成框架设计,及计算中间件 [**Linkis**](https://github.com/WeBankFinTech/Linkis) ,可轻松接入上层各种数据应用系统,让数据开发变得简洁又易用。 -DataSphere Studio定位为数据应用开发门户,闭环涵盖数据应用开发全流程。在统一的UI下,以工作流式的图形化拖拽开发体验,满足从数据导入、脱敏清洗、分析挖掘、质量检测、可视化展现、定时调度到数据输出应用等,数据应用开发全流程场景需求。 +        在统一的 UI 下,DataSphere Studio 以工作流式的图形化拖拽开发体验,将满足从数据交换、脱敏清洗、分析挖掘、质量检测、可视化展现、定时调度到数据输出应用等,数据应用开发全流程场景需求。 -借助于Linkis计算中间件的连接、复用与简化能力,DSS天生便具备了金融级高并发、高可用、多租户隔离和资源管控等执行与调度能力。 +        **DSS 通过插拔式的集成框架设计,让用户可以根据需要,简单快速替换 DSS 已集成的各种功能组件,或新增功能组件。** + +        借助于 [**Linkis**](https://github.com/WeBankFinTech/Linkis) 计算中间件的连接、复用与简化能力,DSS 天生便具备了金融级高并发、高可用、多租户隔离和资源管控等执行与调度能力。 ## 界面预览 -请您耐心等待,加载gif需要一些时间。 +        请您耐心等待,加载 gif 需要一些时间。 ![DSS-V1.0 GIF](images/en_US/readme/DSS_gif.gif) ## 核心特点 -DSS主要特点: +        DSS 主要特点: ### 一、一站式、全流程的应用开发管理界面 -       DSS集成度极高,目前已集成的系统有: +        DSS 集成度极高,目前已集成的系统有(**DSS 对以上组件的版本兼容性,请访问:[已集成组件的兼容性列表](README-ZH.md#四已集成的数据应用组件)**):        1、数据开发IDE工具——[Scriptis](https://github.com/WeBankFinTech/Scriptis) -        2、数据可视化工具——[Visualis](https://github.com/WeBankFinTech/Visualis)(基于宜信[Davinci](https://github.com/edp963/davinci)二次开发) +        2、数据可视化工具——[Visualis](https://github.com/WeBankFinTech/Visualis)(基于宜信 [Davinci](https://github.com/edp963/davinci) 二次开发)        3、数据质量管理工具——[Qualitis](https://github.com/WeBankFinTech/Qualitis) -        4、工作流调度工具——[Azkaban](https://azkaban.github.io/) +        4、工作流调度工具——[Schedulis](https://github.com/WeBankFinTech/Schedulis) + +        5、数据交换工具——[Exchangis](https://github.com/WeBankFinTech/Exchangis) + +        6、数据Api服务——[DataApiService](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E4%BD%BF%E7%94%A8%E6%96%87%E6%A1%A3/DataApiService%E4%BD%BF%E7%94%A8%E6%96%87%E6%A1%A3.md) + +        7、流式应用开发管理工具——[Streamis](https://github.com/WeBankFinTech/Streamis) + +        8、一站式机器学习平台——[Prophecis](https://github.com/WeBankFinTech/Prophecis) + +        9、工作流任务调度平台——[DolphinScheduler](https://github.com/apache/dolphinscheduler) + +        10、数据模型中心——**DataModelCenter**(**社区联合共建中**) + +        **DSS 对以上组件的版本兼容性,请访问:[已集成组件的兼容性列表](README-ZH.md#四已集成的数据应用组件)**。 + +        DSS 插拔式的框架设计模式,允许用户快速替换DSS已集成的各个 Web 系统。如:将 Scriptis 替换成 Zeppelin,将 Schedulis 替换成 DolphinScheduler。 ![DSS一站式](images/zh_CN/readme/onestop.gif) -### 二、基于Linkis计算中间件,打造独有的AppJoint设计理念 +### 二、基于Linkis计算中间件,打造独有的AppConn设计理念 -        AppJoint——应用关节,定义了一套统一的前后台接入规范,可让外部数据应用系统快速简单地接入,成为DSS数据应用开发中的一环。 +        AppConn,是 DSS 可以简单快速集成各种上层 Web 系统的核心概念。 -        DSS通过串联多个AppJoint,编排成一条支持实时执行和定时调度的工作流,用户只需简单拖拽即可完成数据应用的全流程开发。 +        AppConn ——应用连接器,定义了一套统一的前后台接入协议,总共分为三级规范,可让外部数据应用系统快速简单地接入,成为 DSS 数据应用开发中的一环。 + +        AppConn 的三级规范即:一级 SSO 规范,二级组织结构规范,三级开发流程规范; + +        DSS 通过串联多个 AppConn,编排成一条支持实时执行和定时调度的工作流,用户只需简单拖拽即可完成数据应用的全流程开发。 -        由于AppJoint对接了Linkis,外部数据应用系统因此具备了资源管控、并发限流、用户资源管理等能力,且允许上下文信息跨系统级共享,彻底告别应用孤岛。 +        由于 AppConn 对接了 Linkis,外部数据应用系统因此具备了资源管控、并发限流、用户资源管理等能力,且允许上下文信息跨系统级共享,彻底告别应用孤岛。 -### 三、Project级管理单元 +### 三、Workspace级管理单元 -        以Project为管理单元,组织和管理各数据应用系统的业务应用,定义了一套跨数据应用系统的项目协同开发通用标准。 +        以 Workspace 为管理单元,组织和管理各数据应用系统的业务应用,定义了一套跨数据应用系统的工作空间协同开发通用标准,并提供了用户角色管理能力。 ### 四、已集成的数据应用组件 -        1、DSS的调度能力——Azkaban AppJoint - -           用户的很多数据应用,通常希望具备周期性的调度能力。 - -           目前市面上已有的开源调度系统,与上层的其他数据应用系统整合度低,且难以融通。 - -           DSS通过实现Azkaban AppJoint,允许用户将一个编排好的工作流,一键发布到Azkaban中进行定时调度。 - -           DSS还为调度系统定义了一套标准且通用的DSS工作流解析发布规范,让其他调度系统可以轻松与DSS实现低成本对接。 - -![Azkaban](images/zh_CN/readme/Azkaban_AppJoint.gif) - -        2、数据开发——Scriptis AppJoint - -           什么是[Scriptis](https://github.com/WeBankFinTech/Scriptis)? - -           Scriptis是一款支持在线写SQL、Pyspark、HiveQL等脚本,提交给[Linkis](https://github.com/WeBankFinTech/Linkis)执行的数据分析Web工具,且支持UDF、函数、资源管控和智能诊断等企业级特性。 - -           Scriptis AppJoint为DSS集成了Scriptis的数据开发能力,并允许Scriptis的各种脚本类型,作为DSS工作流的节点,参与到应用开发的流程中。 - -           目前已支持HiveSQL、SparkSQL、Pyspark、Scala等脚本节点类型。 - -![Scriptis](images/zh_CN/readme/Scriptis_AppJoint.gif) - -        3、数据可视化——Visualis AppJoint - -           什么是Visualis? - -           Visualis是一个基于宜信开源项目Davinci二次开发的数据可视化BI工具,为用户在数据安全和权限方面,提供金融级数据可视化能力。 - -           Visualis AppJoint为DSS集成了Visualis的数据可视化能力,并允许数据大屏和仪表盘,作为DSS工作流的节点,与上游的数据集市关联起来。 - -![Visualis](images/zh_CN/readme/Visualis_AppJoint.gif) - -        4、数据质量——Qualitis AppJoint - -           Qualitis AppJoint 为DSS集成数据质量校验能力,将数据质量系统集成到DSS工作流开发中,对数据完整性、正确性等进行校验。 - -![Qualitis](images/zh_CN/readme/Qualitis_AppJoint.gif) - -        5、数据发送——Sender AppJoint - -           Sender AppJoint为DSS集成数据发送能力,目前支持SendEmail节点类型,所有其他节点的结果集,都可以通过邮件发送。 - -           例如:SendEmail节点可直接将Display数据大屏作为邮件发送出来。 - -        6、信号节点——Signal AppJoint - -           EventChecker AppJoint用于强化业务与流程之间的解耦和相互关联。 - -           DataChecker节点:检查库表分区是否存在。 - -           EventSender: 跨工作流和工程的消息发送节点。 - -           EventReceiver: 跨工作流和工程的消息接收节点。 - -        7、功能节点 - -           空节点、子工作流节点。 - - -## 与类似系统对比 - -        DSS是一个引领数据应用开发管理方向的开源项目,开源社区目前尚没有同类产品。 +        DSS 通过实现多个 AppConn,已集成了丰富多样的各种上层数据应用系统,基本可满足用户的数据开发需求。 + +        **如果有需要,也可以轻松集成新的数据应用系统,以替换或丰富 DSS 的数据应用开发流程。** [点我了解如何快速集成新的应用系统](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%BC%80%E5%8F%91%E6%96%87%E6%A1%A3/%E7%AC%AC%E4%B8%89%E6%96%B9%E7%B3%BB%E7%BB%9F%E6%8E%A5%E5%85%A5DSS%E5%BC%80%E5%8F%91%E6%8C%87%E5%8D%97.md) + +| 应用工具 | 描述 | DSS0.X 兼容版本(推荐 DSS0.9.1) | DSS1.0 兼容版本(推荐 DSS1.1.0) | +| --------------- | -------------------------------------------------------------------- | --------- | ---------- | +| [**Linkis**](https://github.com/apache/incubator-linkis) | 计算中间件 Apache Linkis,通过提供 REST/WebSocket/JDBC/SDK 等标准接口,上层应用可以方便地连接访问 MySQL/Spark/Hive/Presto/Flink 等底层引擎. | 推荐 Linkis0.11.0(**已发布**) | >= Linkis1.1.1(**已发布**) | +| [**DataApiService**](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E4%BD%BF%E7%94%A8%E6%96%87%E6%A1%A3/DataApiService%E4%BD%BF%E7%94%A8%E6%96%87%E6%A1%A3.md) | (DSS已内置的第三方应用工具)数据API服务。可快速将SQL脚本发布为一个 Restful 接口,对外提供 Rest 访问能力。 | 不支持 | 推荐 DSS1.1.0(**已发布**)| +| [**Scriptis**](https://github.com/WeBankFinTech/DataSphereStudio) | (DSS 已内置的第三方应用工具)支持在线写 SQL、Pyspark、HiveQL 等脚本,提交给 [Linkis](https://github.com/WeBankFinTech/Linkis) 执行的数据分析 Web 工具。 | 推荐 DSS0.9.1(**已发布**) | 推荐 DSS1.1.0(**已发布**) | +| [**Schedulis**](https://github.com/WeBankFinTech/Schedulis) | 基于 Azkaban 二次开发的工作流任务调度系统,具备高性能,高可用和多租户资源隔离等金融级特性。 | 推荐 Schedulis0.6.1(**已发布**) | >= Schedulis0.7.0(**已发布**) | +| **EventCheck** | (DSS 已内置的第三方应用工具)提供跨业务、跨工程和跨工作流的信号通信能力。 | 推荐 DSS0.9.1(**已发布**) | 推荐 DSS1.1.0(**已发布**) | +| **SendEmail** | (DSS 已内置的第三方应用工具)提供数据发送能力,所有其他工作流节点的结果集,都可以通过邮件进行发送 | 推荐 DSS0.9.1(**已发布**) | 推荐 DSS1.1.0(**已发布**) | +| [**Qualitis**](https://github.com/WeBankFinTech/Qualitis) | 数据质量校验工具,提供数据完整性、正确性等数据校验能力 | 推荐 Qualitis0.8.0(**已发布**) | >= Qualitis0.9.2(**已发布**) | +| [**Streamis**](https://github.com/WeBankFinTech/Streamis) | 流式应用开发管理工具。支持发布 Flink Jar 和 Flink SQL ,提供流式应用的开发调试和生产管理能力,如:启停、状态监控、checkpoint 等。 | 不支持 | >= Streamis0.2.0(**已发布**) | +| [**Prophecis**](https://github.com/WeBankFinTech/Prophecis) | 一站式机器学习平台,集成多种开源机器学习框架。Prophecis 的 MLFlow 通过 AppConn 可以接入到 DSS 工作流中。 | 不支持 | >= Prophecis 0.3.2(**已发布**) | +| [**Exchangis**](https://github.com/WeBankFinTech/Exchangis) | 支持对结构化及无结构化的异构数据源之间的数据传输的数据交换平台,即将发布的 Exchangis1.0,将与 DSS 工作流打通 | 不支持 | = Exchangis1.0.0(**已发布**) | +| [**Visualis**](https://github.com/WeBankFinTech/Visualis) | 基于宜信开源项目 Davinci 二次开发的数据可视化 BI 工具,为用户在数据安全方面提供金融级数据可视化能力。 | 推荐 Visualis0.5.0 | = Visualis1.0.0(**已发布**) | +| [**DolphinScheduler**](https://github.com/apache/dolphinscheduler) | Apache DolphinScheduler,分布式易扩展的可视化工作流任务调度平台,支持一键将DSS工作流发布到 DolphinScheduler。 | 不支持 | DolphinScheduler1.3.X(**已发布**) | +| **UserGuide** | (DSS 将内置的第三方应用工具)包含帮助文档、新手指引、Dark模式换肤等。 | 不支持 | >= DSS1.1.0(**已发布**) | +| **DataModelCenter** | (DSS 将内置的第三方应用工具)主要提供数仓规划、数据模型开发和数据资产管理的能力。数仓规划包含主题域、数仓分层、修饰词等;数据模型开发包含指标、维度、度量、向导式建表等;数据资产打通 **Apache Atlas**,提供**数据血缘**能力。 | 不支持 | 规划在 DSS1.2.0(**开发中**) | +| **UserManager** | (DSS 已内置的第三方应用工具)自动初始化一个 DSS 新用户所必须的所有用户环境,包含:创建 Linux 用户、各种用户路径、目录授权等。 | 推荐 DSS0.9.1(**已发布**) | 规划中 | +| [**Airflow**](https://github.com/apache/airflow) | 支持将 DSS 工作流发布到 Apache Airflow 进行定时调度。 | PR 尚未合并 | 不支持 | -## 使用场景 -        DataSphere Studio适用于以下场景: +## Demo试用环境 -        1. 正在筹建或初步具备大数据平台能力,但无任何数据应用工具的场景。 +        由于 DataSphereStudio 支持执行脚本风险较高,WeDataSphere Demo 环境的隔离没有做完,考虑到大家都在咨询 Demo 环境,决定向社区先定向发放邀请码,接受企业和组织的试用申请。 -        2. 已具备大数据基础平台能力,且仅有少数数据应用工具的场景。 +        如果您想试用 Demo 环境,请加入DataSphere Studio社区用户群(**加群方式请翻到本文档末尾处**),联系 **WeDataSphere 入群机器人** 获取邀请码。 -        3. 已具备大数据基础平台能力,且拥有全部数据应用工具,但工具间尚未打通,用户使用隔离感强、学习成本高的场景。 +        DataSphereStudio Demo 环境用户注册页面:[点我进入](https://dss-open.wedatasphere.com/#/register) -        4. 已具备大数据基础平台能力,且拥有全部数据应用工具,部分工具已实现对接,但尚未定义统一规范的场景。 +        DataSphereStudio Demo 环境登录页面:[点我进入](https://dss-open.wedatasphere.com/#/login) -## Demo试用环境 -        [点我进入](https://sandbox.webank.com/wds/dss/#) Demo试用环境 +## 下载 + +        请前往 [DSS releases](https://github.com/WeBankFinTech/DataSphereStudio/releases) 页面下载 DSS 的已编译版本或源码包。 + +## 编译和安装部署 + +        请参照 [编译指引](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%BC%80%E5%8F%91%E6%96%87%E6%A1%A3/DSS%E7%BC%96%E8%AF%91%E6%96%87%E6%A1%A3.md) 来编译 DSS 源码。 + +        请参考 [安装部署文档](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2/DSS%26Linkis%E4%B8%80%E9%94%AE%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3%E5%8D%95%E6%9C%BA%E7%89%88.md) 来部署 DSS。 + +## 示例和使用指引 -        用户/密码: bdp/Abcd1234 +        请到 [用户使用文档](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E4%BD%BF%E7%94%A8%E6%96%87%E6%A1%A3/DSS%E7%94%A8%E6%88%B7%E6%89%8B%E5%86%8C.md) ,了解如何快速使用DSS。 -## 快速安装使用 +## 文档 -点我进入[快速安装使用]() +        DSS1.0 的完整文档列表,请参见 [DSS-Doc](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN) + +        以下为 DSS 相关 AppConn 插件的安装指南: + +- [DSS 的 Visualis AppConn 插件安装指南](https://github.com/WeBankFinTech/Visualis/blob/master/visualis_docs/zh_CN/Visualis_appconn_install_cn.md) + +- [DSS 的 Schedulis AppConn 插件安装指南](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2/SchedulisAppConn%E6%8F%92%E4%BB%B6%E5%AE%89%E8%A3%85%E6%96%87%E6%A1%A3.md) + +- [DSS 的 Qualitis AppConn 插件安装指南](https://github.com/WeBankFinTech/Qualitis/blob/master/docs/zh_CN/ch1/%E6%8E%A5%E5%85%A5%E5%B7%A5%E4%BD%9C%E6%B5%81%E6%8C%87%E5%8D%97.md) + +- [DSS 的 Exchangis AppConn 插件安装指南](https://github.com/WeDataSphere/Exchangis/blob/master/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) + +- [DSS 的 Streamis AppConn 插件安装指南](https://github.com/WeBankFinTech/Streamis/blob/main/docs/zh_CN/0.2.0/development/StreamisAppConn%E5%AE%89%E8%A3%85%E6%96%87%E6%A1%A3.md) + +- [DSS 的 Prophecis AppConn 插件安装指南](https://github.com/WeBankFinTech/Prophecis/blob/master/docs/zh_CN/Deployment_Documents/Prophecis%20Appconn%E5%AE%89%E8%A3%85%E6%96%87%E6%A1%A3.md) + +- [DSS 的 Dolphinscheduler AppConn 插件安装指南](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2/DolphinScheduler%E6%8F%92%E4%BB%B6%E5%AE%89%E8%A3%85%E6%96%87%E6%A1%A3.md) ## 架构 ![DSS架构](images/zh_CN/readme/architecture.png) -## 文档列表 -[DSS编译文档]() +## 使用场景 -[用户手册]() +        DataSphere Studio 适用于以下场景: -[外部系统快速接入DSS]() +        1. 正在筹建或初步具备大数据平台能力,但无任何数据应用工具的场景。 -更多文档,敬请期待! +        2. 已具备大数据基础平台能力,且仅有少数数据应用工具的场景。 -## 交流贡献 +        3. 已具备大数据基础平台能力,且拥有全部数据应用工具,但工具间尚未打通,用户使用隔离感强、学习成本高的场景。 + +        4. 已具备大数据基础平台能力,且拥有全部数据应用工具,部分工具已实现对接,但尚未定义统一规范的场景。 + + +## 贡献 + +        我们非常欢迎和期待更多的贡献者参与共建 DSS, 不论是代码、文档,或是其他能够帮助到社区的贡献形式。 + +## 联系我们 + +        对 DSS 的任何问题和建议,敬请提交 issue,以便跟踪处理和经验沉淀共享。 + +        您也可以扫描下面的二维码,加入我们的 微信/QQ群,以获得更快速的响应。 ![交流](images/zh_CN/readme/communication.png) +## 谁在使用 DSS + +        我们创建了 [Who is using DSS](https://github.com/WeBankFinTech/DataSphereStudio/issues/1) issue 以便用户反馈和记录谁在使用 DSS,欢迎您注册登记. + +        DSS 自2019年开源发布以来,累计已有700多家试验企业和1000+沙盒试验用户,涉及金融、电信、制造、互联网等多个行业。 + ## License -DSS is under the Apache 2.0 license. See the [License](LICENSE) file for details. \ No newline at end of file +        DSS is under the Apache 2.0 license. See the [License](https://github.com/WeBankFinTech/DataSphereStudio/blob/master/LICENSE) file for details. diff --git a/README.md b/README.md index 429deca1d..2dd206c7b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![DSS](images/en_US/readme/DSS.png) +![DSS](images/en_US/readme/DSS_logo.png) ==== [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) @@ -7,17 +7,17 @@ English | [中文](README-ZH.md) ## Introduction -DataSphere Studio (DSS for short) is WeDataSphere, a big data platform of WeBank, a self-developed one-stop data application development management portal. +        DataSphere Studio (DSS for short) is WeDataSphere, a one-stop data application development management portal developed by WeBank. -Based on [Linkis](https://github.com/WeBankFinTech/Linkis) computation middleware, DSS can easily integrate upper-level data application systems, making data application development simple and easy to use. +        With the pluggable integrated framework design and the Linkis, a computing middleware, DSS can easily integrate various upper-layer data application systems, making data development simple and easy to use. -DataSphere Studio is positioned as a data application development portal, and the closed loop covers the entire process of data application development. With a unified UI, the workflow-like graphical drag-and-drop development experience meets the entire lifecycle of data application development from data import, desensitization cleaning, data analysis, data mining, quality inspection, visualization, scheduling to data output applications, etc. +        DataSphere Studio is positioned as a data application development portal, and the closed loop covers the entire process of data application development. With a unified UI, the workflow-like graphical drag-and-drop development experience meets the entire lifecycle of data application development from data import, desensitization cleaning, data analysis, data mining, quality inspection, visualization, scheduling to data output applications, etc. -With the connection, reusability, and simplification capabilities of Linkis, DSS is born with financial-grade capabilities of high concurrency, high availability, multi-tenant isolation, and resource management. +        With the connection, reusability, and simplification capabilities of Linkis, DSS is born with financial-grade capabilities of high concurrency, high availability, multi-tenant isolation, and resource management. ## UI preview -Please be patient, it will take some time to load gif. +        Please be patient, it will take some time to load gif. ![DSS-V1.0 GIF](images/en_US/readme/DSS_gif.gif) @@ -25,137 +25,159 @@ Please be patient, it will take some time to load gif. ### 1. One-stop, full-process application development management UI -       DSS is highly integrated. Currently integrated systems include: - -        a. [Scriptis](https://github.com/WeBankFinTech/Scriptis) - Data Development IDE Tool. - -        b. [Visualis](https://github.com/WeBankFinTech/Visualis) - Data Visualization Tool(Based on the open source project [Davinci](https://github.com/edp963/davinci) contributed by CreditEase) - -        c. [Qualitis](https://github.com/WeBankFinTech/Qualitis) - Data Quality Management Tool +        DSS is highly integrated. Currently integrated components include(**DSS version compatibility for the above components, please visit: [Compatibility list of integrated components](README.md#4-integrated-data-application-components)**): + +        1. Data Development IDE Tool - [Scriptis](https://github.com/WeBankFinTech/Scriptis) + +        2. Data Visualization Tool - [Visualis](https://github.com/WeBankFinTech/Visualis) (Based on the open source project [Davinci](https://github.com/edp963/davinci ) contributed by CreditEase) + +        3. Data Quality Management Tool - [Qualitis](https://github.com/WeBankFinTech/Qualitis) + +        4. Workflow scheduling tool - [Schedulis](https://github.com/WeBankFinTech/Schedulis) + +        5. Data Exchange Tool - [Exchangis](https://github.com/WeBankFinTech/Exchangis) + +        6. Data Api Service - [DataApiService](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/en_US/Using_Document/DataApiService_Usage_Documentation.md) + +        7. Streaming Application Development Management Tool - [Streamis](https://github.com/WeBankFinTech/Streamis) + +        8. One-stop machine Learning Platform - [Prophecis](https://github.com/WeBankFinTech/Prophecis) + +        9. Workflow Task Scheduling Tool - DolphinScheduler (**In Code Merging**) -        d. [Azkaban](https://azkaban.github.io/) - Batch workflow job scheduler +        10. Help documentation and beginner's guide - UserGuide (**In Code Merging**) + +        11. Data Model Center - DataModelCenter (**In development**) +        **DSS version compatibility for the above components, please visit: [Compatibility list of integrated components](README.md#4-integrated-data-application-components)**. + +        With a pluggable framework architecture, DSS is designed to allow users to quickly integrate new data application tools, or replace various tools that DSS has integrated. For example, replace Scriptis with Zeppelin, and replace Schedulis with DolphinScheduler... + ![DSS one-stop video](images/en_US/readme/onestop.gif) -### 2. AppJoint, based on Linkis,defines a unique design concept +### 2. AppConn, based on Linkis,defines a unique design concept + +        AppConn is the core concept that enables DSS to easily and quickly integrate various upper-layer web systems. -        AppJoint——application joint, defining unified front-end and back-end - integration specifications, can quickly and easily integrate with external data application systems, - making them as part of DSS data application development. +        AppConn, an application connector, defines a set of unified front-end and back-end three-level integration protocols, allowing external data application systems to easily and quickly becoming a part of DSS data application development. -        DSS arranges multiple AppJoints in series to form a workflow that supports real-time execution and scheduled execution. Users can complete the entire process development of data applications with simple drag and drop operations. +        The three-level specifications of AppConn are: the first-level SSO specification, the second-level organizational structure specification, and the third-level development process specification. -        Since AppJoint is integrated with Linkis, the external data application system shares the capabilities of resource management, concurrent limiting, and high performance. AppJoint also allows sharable context across system level and completely gets away from application silos. +        DSS arranges multiple AppConns in series to form a workflow that supports real-time execution and scheduled execution. Users can complete the entire process development of data applications with simple drag and drop operations. -### 3. Project, as the management unit +        Since AppConn is integrated with Linkis, the external data application system shares the capabilities of resource management, concurrent limiting, and high performance. AppConn also allows sharable context across system level and thus makes external data application completely gets away from application silos. -        With Project as the management unit, DSS organizes and manages the business applications of each data application system, and defines a set of common standards for collaborative development of projects across data application systems. +### 3. Workspace, as the management unit + +        With Workspace as the management unit, it organizes and manages business applications of various data application systems, defines a set of common standards for collaborative development of workspaces across data application systems, and provides user role management capabilities. ### 4. Integrated data application components -       a. Azkaban AppJoint —— Batch workflow job scheduler - -         Many data applications developed by users usually require periodic scheduling capability. - -         At present, the open source scheduling system in the community is pretty unfriendly to integrate with other data application systems. - -         DSS implements Azkaban AppJoint, which allows users to publish DSS workflows to Azkaban for regular scheduling. - -         DSS also defines standard and generic workflow parsing and publishing specifications for scheduling systems, allowing other scheduling systems to easily achieve low-cost integration with DSS. - -![Azkaban](images/en_US/readme/Azkaban_AppJoint.gif) - -       b. Scriptis AppJoint —— Data Development IDE Tool - -         What is [Scriptis](https://github.com/WeBankFinTech/Scriptis)? - -         Scriptis is for interactive data analysis with script development(SQL, Pyspark, HiveQL), task submission(Spark, Hive), UDF, function, resource management and intelligent diagnosis. - -         Scriptis AppJoint integrates the data development capabilities of Scriptis to DSS, and allows various script types of Scriptis to serve as nodes in the DSS workflow to participate in the application development process. - -         Currently supports HiveSQL, SparkSQL, Pyspark, Scala and other script node types. - -![Scriptis](images/en_US/readme/Scriptis_AppJoint.gif) - -       c. Visualis AppJoint —— Data Visualization Tool - -         What is [Visualis](https://github.com/WeBankFinTech/Visualis)? - -         Visualis is a BI tool for data visualization. It provides financial-grade data visualization capabilities on the basis of data security and permissions, based on the open source project Davinci contributed by CreditEase. - -         Visualis AppJoint integrates data visualization capabilities to DSS, and allows displays and dashboards, as nodes of DSS workflows, to be associated with upstream data market. - -![Visualis](images/en_US/readme/Visualis_AppJoint.gif) - -       d. Qualitis AppJoint —— Data quality management Tool - -         Qualitis AppJoint integrates data quality verification capabilities for DSS, allows Qualitis as a node in DSS workflow - -![Qualitis](images/en_US/readme/Qualitis_AppJoint.gif) - -       e. Data Sender——Sender AppJoint - -         Sender AppJoint provides data delivery capability for DSS. Currently it supports the SendEmail node type, and the result sets of all other nodes can be sent via email. - -         For example, the SendEmail node can directly send the screen shot of a display as an email. - -       f. Signal AppJoint —— Signal Nodes - -         Signal AppJoint is used to strengthen the correlation between business and process while keeping them decoupled. - -         DataChecker Node:Checks whether a table or partition exists. - -         EventSender Node: Messaging nodes across workflows and projects. - -         EventReceiver: Receive nodes for messages across workflows and projects. - -       g. Function node - -         Empty nodes, sub workflow nodes. - -## Compared with similar systems - -       DSS is an open source project leading the direction of data application development and management. - The open source community currently does not have similar products. +        DSS has integrated a variety of upper-layer data application systems by implementing multiple AppConns, which can basically meet the data development needs of users. -## Usage Scenarios +        **If desired, new data application systems can also be easily integrated to replace or enrich DSS's data application development process.** [Click me to learn how to quickly integrate new application systems](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/en_US/Development_Documentation/Third-party_System_Access_Development_Guide.md) -       DataSphere Studio is suitable for the following scenarios: +|Component | Description | DSS0.X compatible version (DSS0.9.1 recommended) | DSS1.0 compatible version (DSS1.1.0 recommended) | +| --------------- | -------------------------------------------------------------------- | --------- | ---------- | +| [**Linkis**](https://github.com/apache/incubator-linkis) | Computing middleware Apache Linkis, by providing standard interfaces such as REST/WebSocket/JDBC/SDK, upper-layer applications can easily connect and access underlying engines such as MySQL/Spark/Hive/Presto/Flink. | Linkis0.11.0 is recommended (**Released* *) | >= Linkis1.1.1 (**released**) | +| [**DataApiService**](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/en_US/Using_Document/DataApiService_Usage_Documentation.md) | (DSS has built-in third-party application tools) data API service. The SQL script can be quickly published as a Restful interface, providing Rest access capability to the outside world. | Not supported | DSS1.1.0 recommended (**released**)| +| [**Scriptis**](https://github.com/WeBankFinTech/DataSphereStudio) | (DSS has built-in third-party application tools) support online writing of SQL, Pyspark, HiveQL and other scripts, and submit to [Linkis](https ://github.com/WeBankFinTech/Linkis) data analysis web tool. | Recommended DSS0.9.1 (**Released**) | Recommended DSS1.1.0 (**Released**) | +| [**Schedulis**](https://github.com/WeBankFinTech/Schedulis) | Workflow task scheduling system based on Azkaban secondary development, with financial-grade features such as high performance, high availability and multi-tenant resource isolation. | Recommended Schedulis0.6.1 (**released**) | >= Schedulis0.7.0 (**Released**) | +| **EventCheck** | (a third-party application tool built into DSS) provides signal communication capabilities across business, engineering, and workflow. | Recommended DSS0.9.1 (**Released**) | Recommended DSS1.1.0 (**Released**) | +| **SendEmail** | (DSS has built-in third-party application tools) provides the ability to send data, all the result sets of other workflow nodes can be sent by email | DSS0.9.1 is recommended (**released**) | Recommended DSS1.1.0 (**Released**) | +| [**Qualitis**](https://github.com/WeBankFinTech/Qualitis) | Data quality verification tool, providing data verification capabilities such as data integrity and correctness | Qualitis0.8.0 is recommended (**Released **) | >= Qualitis0.9.2 (**Released**) | +| [**Streamis**](https://github.com/WeBankFinTech/Streamis) | Streaming application development management tool. It supports the release of Flink Jar and Flink SQL, and provides the development, debugging and production management capabilities of streaming applications, such as: start-stop, status monitoring, checkpoint, etc. | Not supported | >= Streamis0.2.0 (**Released**) | +| [**Prophecis**](https://github.com/WeBankFinTech/Prophecis) | A one-stop machine learning platform that integrates multiple open source machine learning frameworks. Prophecis' MLFlow can be connected to DSS workflow through AppConn. | Not supported | >= Prophecis 0.3.2 (**Released**) | +| [**Exchangis**](https://github.com/WeBankFinTech/Exchangis) | A data exchange platform that supports data transmission between structured and unstructured heterogeneous data sources, the upcoming Exchangis1. 0, will work with DSS workflow | not supported | = Exchangis1.0.0 (**Released**) | +| [**Visualis**](https://github.com/WeBankFinTech/Visualis) | A data visualization BI tool based on the secondary development of Davinci, an open source project of CreditEase, provides users with financial-level data visualization capabilities in terms of data security. | Recommended Visualis0.5.0 |= Visualis1.0.0 (**Released**) | +| [**DolphinScheduler**](https://github.com/apache/dolphinscheduler) | Apache DolphinScheduler, a distributed and easily scalable visual workflow task scheduling platform, supports one-click publishing of DSS workflows to DolphinScheduler. | Not supported | DolphinScheduler1.3.X (**Released**) | +| **UserGuide** | (DSS will be built-in third-party application tools) contains help documents, beginner's guide, Dark mode skinning, etc. | Not supported | >= DSS1.1.0 (**Released**) | +| **DataModelCenter** | (the third-party application tool that DSS will build) mainly provides data warehouse planning, data model development and data asset management capabilities. Data warehouse planning includes subject domains, data warehouse hierarchies, modifiers, etc.; data model development includes indicators, dimensions, metrics, wizard-based table building, etc.; data assets are connected to Apache Atlas to provide data lineage capabilities . | Not supported | Planned in DSS1.2.0 (**under development**) | +| **UserManager** | (DSS has built-in third-party application tools) automatically initialize all user environments necessary for a new DSS user, including: creating Linux users, various user paths, directory authorization, etc. | Recommended DSS0.9.1 (**Released**) | Planning | +| [**Airflow**](https://github.com/apache/airflow) | Supports publishing DSS workflows to Apache Airflow for scheduled scheduling. | PR not yet merged | Not supported | -       1. Scenarios in which big data platform capability is being prepared or initialized but no data application tools are available. -       2. Scenarios in which users already have big data foundation platform capabilities but with only a few data application tools. +## Demo Trial environment -       3. Scenarios in which users have the ability of big data foundation platform and comprehensive data application tools, but suffers strong isolation and and high learning costs because those tools have not been integrated together. +        The function of DataSphere Studio supporting script execution has high security risks, and the isolation of the WeDataSphere Demo environment has not been completed. Considering that many users are inquiring about the Demo environment, we decided to first issue invitation codes to the community and accept trial applications from enterprises and organizations. -       4. Scenarios in which users have the capabilities of big data foundation platform and comprehensive data application tools. but lacks unified and standardized specifications, while a part of these tools have been integrated. +        If you want to try out the Demo environment, please join the DataSphere Studio community user group (**Please refer to the end of the document**), and contact **WeDataSphere Group Robot** to get an invitation code. + +        DataSphereStudio Demo environment user registration page: [click me to enter](https://dss-open.wedatasphere.com/#/register) + +        DataSphereStudio Demo environment login page: [click me to enter](https://dss-open.wedatasphere.com/#/login) + +## Download + +        Please go to the [DSS Releases Page](https://github.com/WeBankFinTech/DataSphereStudio/releases) to download a compiled version or a source code package of DSS. + +## Compile and deploy -## Demo environment +        Please follow [Compile Guide](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/en_US/Development_Documentation/Compilation_Documentation.md) to compile DSS from source code. -       [Click me to](https://sandbox.webank.com/wds/dss/#) DSS Demo environment +        Please refer to [Deployment Documents](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/en_US/Installation_and_Deployment/DSS%26Linkis_one-click_deployment_document_stand-alone_version.md) to do the deployment. -       User/Password: bdp/Abcd1234 +## Examples and Guidance -## Quick start +        You can find examples and guidance for how to use DSS in [User Manual](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/en_US/Using_Document/DSS_User_Manual.md). -Click to [Quick start]() + +## Documents + +        For a complete list of documents for DSS1.0, see [DSS-Doc](https://github.com/WeBankFinTech/DataSphereStudio-Doc) + +        The following is the installation guide for DSS-related AppConn plugins: + +- [Visualis AppConn Plugin Installation Guide for DSS](https://github.com/WeBankFinTech/Visualis/blob/master/visualis_docs/en_US/Visualis_appconn_install_en.md) + +- [Schedulis AppConn Plugin Installation Guide for DSS](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/en_US/Installation_and_Deployment/Schedulis_Linkis_JobType_Installation_Documentation.md) + +- [Qualitis AppConn Plugin Installation Guide for DSS](https://github.com/WeBankFinTech/Qualitis/blob/master/docs/zh_CN/ch1/%E6%8E%A5%E5%85%A5%E5%B7%A5%E4%BD%9C%E6%B5%81%E6%8C%87%E5%8D%97.md) + +- [Exchangis AppConn Plugin Installation Guide for DSS](https://github.com/WeDataSphere/Exchangis/blob/master/docs/en_US/ch1/exchangis_appconn_deploy_en.md) + +- [Streamis AppConn Plugin Installation Guide for DSS](https://github.com/WeBankFinTech/Streamis/blob/main/docs/en_US/0.2.0/development/StreamisAppConnInstallationDocument.md) + +- [Prophecis AppConn Plugin Installation Guide for DSS](https://github.com/WeBankFinTech/Prophecis/blob/master/docs/zh_CN/Deployment_Documents/Prophecis%20Appconn%E5%AE%89%E8%A3%85%E6%96%87%E6%A1%A3.md) + +- [DolphinScheduler AppConn Plugin Installation Guide for DSS](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/en_US/Installation_and_Deployment/DolphinScheduler_Plugin_Installation_Documentation.md) ## Architecture ![DSS Architecture](images/en_US/readme/architecture.png) -## Documents +## Usage Scenarios + +       DataSphere Studio is suitable for the following scenarios: + +       1. Scenarios in which big data platform capability is being prepared or initialized but no data application tools are available. + +       2. Scenarios in which users already have big data foundation platform capabilities but with only a few data application tools. + +       3. Scenarios in which users have the ability of big data foundation platform and comprehensive data application tools, but suffers strong isolation and and high learning costs because those tools have not been integrated together. -[Compiled documentation]() +       4. Scenarios in which users have the capabilities of big data foundation platform and comprehensive data application tools. but lacks unified and standardized specifications, while a part of these tools have been integrated. + +## Contributing -[User manual]() +        Contributions are always welcomed, we need more contributors to build DSS together. either code, or doc, or other supports that could help the community. -[Quick integration with DSS for external systems]() +        For code and documentation contributions, please follow the contribution guide. ## Communication +        For any questions or suggestions, please kindly submit an issue. + +        You can scan the QR code below to join our WeChat and QQ group to get more immediate response. + ![communication](images/en_US/readme/communication.png) +## Who is using DSS + +        We opened an issue for users to feedback and record who is using DSS. + +        Since the first release of DSS in 2019, it has accumulated more than 700 trial companies and 1000+ sandbox trial users, which involving diverse industries, from finance, banking, tele-communication, to manufactory, internet companies and so on. + ## License -DSS is under the Apache 2.0 license. See the [License](LICENSE) file for details. \ No newline at end of file +        DSS is under the Apache 2.0 license. See the [License](LICENSE) file for details. diff --git a/assembly/bin/appconn-install.sh b/assembly/bin/appconn-install.sh new file mode 100644 index 000000000..b8bec4d67 --- /dev/null +++ b/assembly/bin/appconn-install.sh @@ -0,0 +1,125 @@ +#!/bin/sh +#Actively load user env +source ~/.bashrc +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` + +SOURCE_ROOT=${workDir} + +#load config +source ${SOURCE_ROOT}/conf/config.sh +source ${SOURCE_ROOT}/conf/db.sh + +APPCONN_NAME='' +APPCONN_INSTALL_IP=127.0.0.1 +APPCONN_INSTALL_PORT=8088 + +#echo "Current path of init sql is ${DB_DML_PATH}" +LOCAL_IP="`ifconfig | grep 'inet' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $2}'`" + +function isSuccess(){ + if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 + else + echo "Succeed to " + $1 + fi +} + +PROC_NAME=DSSProjectServerApplication +ProcNumber=`ps -ef | grep -w $PROC_NAME | grep -v grep | wc -l` +if [ $ProcNumber -le 0 ];then + echo "${PROC_NAME} is not running, please ensure whether DSS is installed and started." + exit 10 +else + echo "Begine to install new AppConn plugin..." +fi + +##choose install mysql mode +function initInstallAppConn() { + if [ ! -z $1 ];then + APPCONN_NAME=$1 + echo "Begin to install default appconn:$APPCONN_NAME" + else + echo "Please input the name of installation AppConn, e.g: schedulis." + read -p "Please input the AppConn name:" idx + if [[ 'exit' = "$idx" ]];then + echo "exit!" + exit 1 + else + APPCONN_NAME=$idx + fi + echo "Current installation AppConn is ${APPCONN_NAME}" + echo "" + echo "Please input the installed IP address of $APPCONN_NAME." + echo "e.g: if you have installed $APPCONN_NAME with 192.168.1.1:8080, please input ip in 192.168.1.1" + read -p "Please input the IP: " ip + APPCONN_INSTALL_IP=$ip + echo "" + echo "e.g: if you have installed $APPCONN_NAME with 192.168.1.1:8080, please input port in 8080" + read -p "Please input the port:" port + APPCONN_INSTALL_PORT=$port + fi + + echo "" + replaceCommonIp + echo "The base url of $APPCONN_NAME is http://${APPCONN_INSTALL_IP}:${APPCONN_INSTALL_PORT}/." +} + +function replaceCommonIp() { + if [[ $APPCONN_INSTALL_IP == "127.0.0.1" ]] || [[ $APPCONN_INSTALL_IP == "0.0.0.0" ]];then + echo "since you input the ip to $APPCONN_INSTALL_IP, we will change it to real ip $LOCAL_IP." + APPCONN_INSTALL_IP=$LOCAL_IP + fi +} + +function replaceDefaultAppconnSql() { + sed -i "s#EVENTCHECKER_JDBC_URL#$EVENTCHECKER_JDBC_URL#g" $DB_DML_PATH + sed -i "s#EVENTCHECKER_JDBC_USERNAME#$EVENTCHECKER_JDBC_USERNAME#g" $DB_DML_PATH + sed -i "s#EVENTCHECKER_JDBC_PASSWORD#$EVENTCHECKER_JDBC_PASSWORD#g" $DB_DML_PATH + + sed -i "s#DATACHECKER_JOB_JDBC_URL#$DATACHECKER_JOB_JDBC_URL#g" $DB_DML_PATH + sed -i "s#DATACHECKER_JOB_JDBC_USERNAME#$DATACHECKER_JOB_JDBC_USERNAME#g" $DB_DML_PATH + sed -i "s#DATACHECKER_JOB_JDBC_PASSWORD#$DATACHECKER_JOB_JDBC_PASSWORD#g" $DB_DML_PATH + + sed -i "s#DATACHECKER_BDP_JDBC_URL#$DATACHECKER_BDP_JDBC_URL#g" $DB_DML_PATH + sed -i "s#DATACHECKER_BDP_JDBC_USERNAME#$DATACHECKER_BDP_JDBC_USERNAME#g" $DB_DML_PATH + sed -i "s#DATACHECKER_BDP_JDBC_PASSWORD#$DATACHECKER_BDP_JDBC_PASSWORD#g" $DB_DML_PATH + + sed -i "s#BDP_MASK_IP#127.0.0.1#g" $DB_DML_PATH + sed -i "s#BDP_MASK_PORT#8087#g" $DB_DML_PATH + + sed -i "s#EMAIL_HOST#${EMAIL_HOST}#g" $DB_DML_PATH + sed -i "s#EMAIL_PORT#${EMAIL_PORT}#g" $DB_DML_PATH + sed -i "s#EMAIL_USERNAME#${EMAIL_USERNAME}#g" $DB_DML_PATH + sed -i "s#EMAIL_PASSWORD#${EMAIL_PASSWORD}#g" $DB_DML_PATH + sed -i "s#EMAIL_PROTOCOL#${EMAIL_PROTOCOL}#g" $DB_DML_PATH +} + +##choose execute mysql mode +function executeSQL() { + TEMP_DB_DML_PATH=${SOURCE_ROOT}/dss-appconns/${APPCONN_NAME}/db + DB_DML_PATH=$TEMP_DB_DML_PATH/init_real.sql + cp -rf $TEMP_DB_DML_PATH/init.sql $DB_DML_PATH + sed -i "s/APPCONN_INSTALL_IP/$APPCONN_INSTALL_IP/g" $DB_DML_PATH + sed -i "s/APPCONN_INSTALL_PORT/$APPCONN_INSTALL_PORT/g" $DB_DML_PATH + sed -i "s#DSS_INSTALL_HOME_VAL#$DSS_INSTALL_HOME#g" $DB_DML_PATH + if [ ! -z $1 ];then + replaceDefaultAppconnSql + fi + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH" + isSuccess "initialize the database settings $DB_DML_PATH" + echo "initialize the $TEMP_DB_DML_PATH/init.sql for $APPCONN_NAME succeed." +} + +echo "" +echo "Step1: get the AppConn basic settings." +initInstallAppConn $1 +echo "" + +echo "Step2: initialize the database settings." +executeSQL $1 +echo "" + +echo "Step3: load the plugin of $APPCONN_NAME AppConn in DSS." +sh $SOURCE_ROOT/bin/appconn-refresh.sh $APPCONN_NAME \ No newline at end of file diff --git a/assembly/bin/appconn-refresh.sh b/assembly/bin/appconn-refresh.sh new file mode 100644 index 000000000..7f455a3ee --- /dev/null +++ b/assembly/bin/appconn-refresh.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +if [ -z $SOURCE_ROOT ]; then + #Actively load user env + source ~/.bashrc + shellDir=`dirname $0` + workDir=`cd ${shellDir}/..;pwd` + SOURCE_ROOT=${workDir} + #load config + source ${SOURCE_ROOT}/conf/config.sh + source ${SOURCE_ROOT}/conf/db.sh +fi + +function isSuccess(){ + if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 + else + echo "Succeed to " + $1 + fi +} + +if [ -z $1 ];then + if [ -z $APPCONN_NAME ]; then + APPCONN_NAME='' + echo "Please input the name of refreshing AppConn, e.g: schedulis." + read -p "Please input the AppConn name: " APPCONN_NAME + fi + + echo "" + echo "Try to refresh the plugin of $APPCONN_NAME AppConn in all DSS micro-services." + echo "The following 2 ways can take effect:" + echo "1. restart DSS, we will use ${SOURCE_ROOT}/sbin/dss-start-all.sh to restart it!" + echo "2. do nothing, just wait for 5 minutes. Since the DSS micro-services will refresh all the AppConn plugins every 10 minutes." + echo "" + read -p "Please input the choise: " choise + if [[ '1' = "$choise" ]]; then + echo "You chose to restart DSS, now try to restart DSS..." + sh $SOURCE_ROOT/sbin/dss-start-all.sh + else + echo "You chose to wait for 5 minutes." + echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn." + curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load + isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project." + echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin." + echo "" + exit 0 + fi +else + APPCONN_NAME=$1 + echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn." + curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load + isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project." + echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin." + echo "" + exit 0 +fi \ No newline at end of file diff --git a/assembly/bin/checkEnv.sh b/assembly/bin/checkEnv.sh new file mode 100644 index 000000000..39f546d26 --- /dev/null +++ b/assembly/bin/checkEnv.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +say() { + printf 'check command fail \n %s\n' "$1" +} + +err() { + say "$1" >&2 + exit 1 +} + +check_cmd() { + command -v "$1" > /dev/null 2>&1 +} + +need_cmd() { + if ! check_cmd "$1"; then + err "need '$1' (your linux command not found)" + fi +} +echo "<-----start to check used cmd---->" +need_cmd yum +need_cmd java +need_cmd mysql +need_cmd telnet +need_cmd tar +need_cmd sed +need_cmd dos2unix +echo "<-----end to check used cmd---->" diff --git a/assembly/bin/executeSQL.sh b/assembly/bin/executeSQL.sh new file mode 100644 index 000000000..8b60d0f08 --- /dev/null +++ b/assembly/bin/executeSQL.sh @@ -0,0 +1,88 @@ +#!/bin/sh + +function checkExternalServer(){ + echo "telnet check for your $SERVER_NAME, if you wait for a long time,may be your $SERVER_NAME does not prepared" + result=`echo -e "\n" | telnet $EXTERNAL_SERVER_IP $EXTERNAL_SERVER_PORT 2>/dev/null | grep Connected | wc -l` + if [ $result -eq 1 ]; then + echo "$SERVER_NAME is OK." + else + echo "$SERVER_NAME is Bad. You need to prepare the' $SERVER_NAME ' environment in advance" + exit 1 + fi +} + +## choose install mode +function chooseInstallMode() { + echo "Simple installation mode" + #check for Java + checkJava + #check for mysql + SERVER_NAME=MYSQL + EXTERNAL_SERVER_IP=$MYSQL_HOST + EXTERNAL_SERVER_PORT=$MYSQL_PORT + checkExternalServer +} + +##choose install mysql mode +function chooseInstallMySQLMode() { + echo "Do you want to clear Dss table information in the database?" + echo " 1: Do not execute table-building statements" + echo " 2: Dangerous! Clear all data and rebuild the tables." + echo "" + MYSQL_INSTALL_MODE=1 + read -p "Please input the choice:" idx + if [[ '2' = "$idx" ]];then + MYSQL_INSTALL_MODE=2 + echo "You chose Rebuild the table" + elif [[ '1' = "$idx" ]];then + MYSQL_INSTALL_MODE=1 + echo "You chose not execute table-building statements" + else + echo "no choice,exit!" + exit 1 + fi + + ##init db + if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then + ENV_FLAG="dev" + DB_CONF_PATH=${workDir}/db + DB_DML_PATH=$DB_CONF_PATH/dss_dml_real.sql + replaceAppConnInstanceSQL + executeSQL + fi +} + +##choose execute mysql mode +function executeSQL() { + chooseInstallMode + + sed -i "s/GATEWAY_INSTALL_IP/$GATEWAY_INSTALL_IP/g" $DB_DML_PATH + sed -i "s/GATEWAY_PORT/$GATEWAY_PORT/g" $DB_DML_PATH + + sed -i "s#DSS_INSTALL_HOME_VAL#$DSS_INSTALL_HOME#g" $DB_DML_PATH + + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/dss_ddl.sql" + isSuccess "source dss_ddl.sql" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/dss_apiservice_ddl.sql" + isSuccess "source dss_apiservice_ddl.sql" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/dss_dataapi_ddl.sql" + isSuccess "source dss_dataapi_ddl.sql" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/dss_guide_ddl.sql" + isSuccess "source dss_guide_ddl.sql" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH" + isSuccess "source dss_dml_real.sql" + echo "Rebuild the table" +} + +function replaceAppConnInstanceSQL() { + DB_DML_PATH=$DB_CONF_PATH/dss_dml_real.sql + cp -rf $DB_CONF_PATH/dss_dml.sql $DB_DML_PATH +# sed -i "s#ORCHESTRATOR_IP#$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP#g" $DB_DML_PATH +# sed -i "s#ORCHESTRATOR_PORT#$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT#g" $DB_DML_PATH + + sed -i "s#WORKFLOW_IP#$DSS_WORKFLOW_SERVER_INSTALL_IP#g" $DB_DML_PATH + sed -i "s#WORKFLOW_PORT#$DSS_WORKFLOW_SERVER_PORT#g" $DB_DML_PATH + +} + +chooseInstallMySQLMode diff --git a/assembly/bin/install-default-appconn.sh b/assembly/bin/install-default-appconn.sh new file mode 100644 index 000000000..bf969bd86 --- /dev/null +++ b/assembly/bin/install-default-appconn.sh @@ -0,0 +1,39 @@ +#!/bin/sh +#Actively load user env + +#source ~/.bash_profile + +if [ -f "~/.bashrc" ];then + echo "Warning! user bashrc file does not exist." +else + source ~/.bashrc +fi + +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` + +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 +else + echo "Succeed to" + $1 +fi +} + + +#######设置为当前路径,如果不需要直接注掉这执行函数########## +setCurrentRoot + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to install DSS Default Appconn ######################" +echo "########################################################################" +echo "now begin to install default appconn: datachecker." +sh ${workDir}/bin/appconn-install.sh datachecker +echo "now begin to install default appconn: eventchecker." +sh ${workDir}/bin/appconn-install.sh eventchecker +echo "now begin to install default appconn: sendemail." +sh ${workDir}/bin/appconn-install.sh sendemail +isSuccess "install DSS Default Appconn" \ No newline at end of file diff --git a/assembly/bin/install.sh b/assembly/bin/install.sh new file mode 100644 index 000000000..bf49bea92 --- /dev/null +++ b/assembly/bin/install.sh @@ -0,0 +1,378 @@ +#!/bin/sh +#Actively load user env +if [ -f "~/.bashrc" ];then + echo "Warning! user bashrc file does not exist." +else + source ~/.bashrc +fi + +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` + +SERVER_IP="" +SERVER_HOME="" + +local_host="`hostname --fqdn`" +LOCAL_IP=$(hostname -I) +LOCAL_IP=${LOCAL_IP// /} + +#To be compatible with MacOS and Linux +txt="" +if [[ "$OSTYPE" == "darwin"* ]]; then + txt="''" +elif [[ "$OSTYPE" == "linux-gnu" ]]; then + # linux + txt="" +elif [[ "$OSTYPE" == "cygwin" ]]; then + echo "dss not support Windows operating system" + exit 1 +elif [[ "$OSTYPE" == "msys" ]]; then + echo "dss not support Windows operating system" + exit 1 +elif [[ "$OSTYPE" == "win32" ]]; then + echo "dss not support Windows operating system" + exit 1 +elif [[ "$OSTYPE" == "freebsd"* ]]; then + txt="" +else + echo "Operating system unknown, please tell us(submit issue) for better service" + exit 1 +fi + +function isSuccess(){ + if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 + else + echo "Succeed to " + $1 + fi +} + +function checkJava(){ + java -version + isSuccess "execute java --version" +} + +checkJava + +dos2unix -q ${workDir}/config/* +isSuccess "execute dos2unix -q ${workDir}/config/*" +dos2unix -q ${workDir}/bin/* + +echo "step1:load config" +source ${workDir}/config/config.sh +source ${workDir}/config/db.sh + +DSS_FILE_PATH="$workDir/$DSS_FILE_NAME" +#dos2unix ${DSS_FILE_PATH}/sbin/* +#dos2unix ${DSS_FILE_PATH}/sbin/ext/* +if [ -z $DSS_FILE_NAME ]; then + echo "DSS_FILE_NAME is null " + exit 1 +fi + +function replaceCommonIp() { + if [ -z "$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP" ]; then + DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_FRAMEWORK_PROJECT_SERVER_PORT" ]; then + DSS_FRAMEWORK_PROJECT_SERVER_PORT=9002 + fi + + if [ -z "$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP" ]; then + DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT" ]; then + DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT=9003 + fi + + if [ -z "$DSS_APISERVICE_SERVER_INSTALL_IP" ]; then + DSS_APISERVICE_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_APISERVICE_SERVER_PORT" ]; then + DSS_APISERVICE_SERVER_PORT=9004 + fi + + if [ -z "$DSS_WORKFLOW_SERVER_INSTALL_IP" ]; then + DSS_WORKFLOW_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_WORKFLOW_SERVER_PORT" ]; then + DSS_WORKFLOW_SERVER_PORT=9005 + fi + + if [ -z "$DSS_FLOW_EXECUTION_SERVER_INSTALL_IP" ]; then + DSS_FLOW_EXECUTION_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_FLOW_EXECUTION_SERVER_PORT" ]; then + DSS_FLOW_EXECUTION_SERVER_PORT=9006 + fi + + if [ -z "$DSS_SCRIPTIS_SERVER_INSTALL_IP" ]; then + DSS_SCRIPTIS_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_SCRIPTIS_SERVER_PORT" ]; then + DSS_SCRIPTIS_SERVER_PORT=9008 + fi + + if [ -z "$DSS_DATA_API_SERVER_INSTALL_IP" ]; then + DSS_DATA_API_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_DATA_API_SERVER_PORT" ]; then + DSS_DATA_API_SERVER_PORT=9208 + fi + + if [ -z "$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP" ]; then + DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_DATA_GOVERNANCE_SERVER_PORT" ]; then + DSS_DATA_GOVERNANCE_SERVER_PORT=9209 + fi + + if [ -z "$DSS_GUIDE_SERVER_INSTALL_IP" ]; then + DSS_GUIDE_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_GUIDE_SERVER_PORT" ]; then + DSS_GUIDE_SERVER_PORT=9210 + fi + + if [[ $GATEWAY_INSTALL_IP == "127.0.0.1" ]] || [ -z "$GATEWAY_INSTALL_IP" ]; then + #echo "GATEWAY_INSTALL_IP is equals $GATEWAY_INSTALL_IP ,we will change it to ip address" + GATEWAY_INSTALL_IP=$LOCAL_IP + fi + if [[ $EUREKA_INSTALL_IP == "127.0.0.1" ]] || [ -z "$EUREKA_INSTALL_IP" ]; then + #echo "EUREKA_INSTALL_IP is equals $EUREKA_INSTALL_IP ,we will change it to ip address" + EUREKA_INSTALL_IP=$LOCAL_IP + fi +} +##替换真实的IP +replaceCommonIp + +EUREKA_URL=http://$EUREKA_INSTALL_IP:$EUREKA_PORT/eureka/ + +## excecute sql +source ${workDir}/bin/executeSQL.sh + +function changeCommonConf(){ + sed -i "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $CONF_APPLICATION_YML + sed -i "s#hostname:.*#hostname: $SERVER_IP#g" $CONF_APPLICATION_YML + sed -i "s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT/#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.gateway.wtss.url.*#wds.linkis.gateway.wtss.url=http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT/#g" $CONF_DSS_PROPERTIES +} + +##function start +function changeConf(){ + sed -i "s#spring.server.port=.*#spring.server.port=$SERVER_PORT#g" $CONF_SERVER_PROPERTIES + if [[ $SERVER_NAME == "dss-framework-orchestrator-server" ]] || [[ $SERVER_NAME == "dss-workflow-server" ]]; then + SERVER_FULL_NAME=$SERVER_NAME + SERVER_FULL_NAME=$SERVER_NAME-$ENV_FLAG + sed -i "s#spring.spring.application.name=.*#spring.spring.application.name=$SERVER_FULL_NAME#g" $CONF_SERVER_PROPERTIES + fi + sed -i "s#wds.dss.appconn.scheduler.project.store.dir.*#wds.dss.appconn.scheduler.project.store.dir=$WDS_SCHEDULER_PATH#g" $CONF_SERVER_PROPERTIES + isSuccess "subsitution $CONF_SERVER_PROPERTIES" +} +##function end + + +UPLOAD_PUBLIC_IPS="" +##function start +function uploadServiceFile(){ + if [[ $SERVER_IP == "127.0.0.1" ]]; then + SERVER_IP=$local_host + fi + #echo "$SERVER_NAME-step3:copy install package" + # upload project conf + # cp -rfp $SSH_PORT ${workDir}/config/{$SERVER_NAME}.properties $CONF_PATH + if [[ $UPLOAD_PUBLIC_IPS == *",${ENV_FLAG}-$SERVER_IP,"* ]]; then + return 0 + fi + cp -rfp ${DSS_FILE_PATH}/* $SERVER_HOME + cp -rfp ${workDir}/bin $SERVER_HOME + cp -rfp ${workDir}/config/* $SERVER_HOME/conf + sudo chown -R $deployUser:$deployUser $SERVER_HOME + UPLOAD_PUBLIC_IPS="$UPLOAD_PUBLIC_IPS,${ENV_FLAG}-$SERVER_IP," + changeCommonConf +# echo "UPLOAD_PUBLIC_IPS-->$UPLOAD_PUBLIC_IPS" +} + +##function start +function installPackage(){ + if [[ $SERVER_IP == "127.0.0.1" ]]; then + SERVER_IP=$local_host + fi + if [ -z $SERVER_NAME ]; then + echo "ERROR:SERVER_NAME is null " + exit 1 + fi + uploadServiceFile + # change configuration + changeConf +} + +function dssWebInstall(){ +if ! test -e ${LINKIS_DSS_HOME}/wedatasphere-dss-web*.zip; then + echo "**********Error: please put wedatasphere-dss-web-xxx.zip in ${LINKIS_DSS_HOME}! " + exit 1 +else + echo "Start to unzip dss web package." + unzip -d ${LINKIS_DSS_HOME}/web/ -o ${LINKIS_DSS_HOME}/wedatasphere-dss-web-*.zip > /dev/null 2>&1 + sed -i "s#linkis_url.*#linkis_url=${LINKIS_GATEWAY_URL}#g" ${LINKIS_DSS_HOME}/web/config.sh + isSuccess "Unzip dss web package to ${LINKIS_DSS_HOME}/web" +fi +} + +##Install dss projects +function installDssProject() { + echo "step2:update config" +# if [ "$DSS_INSTALL_HOME" != "" ] +# then +# rm -rf $DSS_INSTALL_HOME +# fi + #echo "" + #echo "-----------------DSS install start--------------------" + SERVER_HOME=$DSS_INSTALL_HOME + if [ "$SERVER_HOME" == "" ] + then + export SERVER_HOME=${workDir}/DSSInstall + fi + if [ -d $SERVER_HOME ] && [ "$SERVER_HOME" != "$workDir" ]; then + rm -r $SERVER_HOME-bak + echo "mv $SERVER_HOME $SERVER_HOME-bak" + mv $SERVER_HOME $SERVER_HOME-bak + fi + echo "create dir SERVER_HOME: $SERVER_HOME" + sudo mkdir -p $SERVER_HOME + isSuccess "Create the dir of $SERVER_HOME" + sudo chown -R $deployUser:$deployUser $SERVER_HOME + isSuccess "chown -R $deployUser:$deployUser $SERVER_HOME" + + #echo "" + SERVER_NAME=dss-framework-project-server + SERVER_IP=$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP + SERVER_PORT=$DSS_FRAMEWORK_PROJECT_SERVER_PORT + + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-framework/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-framework + LOG_PATH=$SERVER_HOME/logs/dss-framework/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install project-Server + installPackage + #echo "" + + SERVER_NAME=dss-framework-orchestrator-server + SERVER_IP=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP + SERVER_PORT=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-framework/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-framework + LOG_PATH=$SERVER_HOME/logs/dss-framework/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install orchestrator-Server + installPackage + #echo "" + + SERVER_NAME=dss-apiservice-server + SERVER_IP=$DSS_APISERVICE_SERVER_INSTALL_IP + SERVER_PORT=$DSS_APISERVICE_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-apps/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-apps + LOG_PATH=$SERVER_HOME/logs/dss-apps/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install dss-apiservice-server + installPackage + #echo "" + + SERVER_NAME=dss-scriptis-server + SERVER_IP=$DSS_SCRIPTIS_SERVER_INSTALL_IP + SERVER_PORT=$DSS_SCRIPTIS_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-apps/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-apps + LOG_PATH=$SERVER_HOME/logs/dss-apps/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install dss-scriptis-server + installPackage + #echo "" + + ##Flow execution Install + PACKAGE_DIR=dss + SERVER_NAME=dss-flow-execution-server + SERVER_IP=$DSS_FLOW_EXECUTION_SERVER_INSTALL_IP + SERVER_PORT=$DSS_FLOW_EXECUTION_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-orchestrator/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-orchestrator + LOG_PATH=$SERVER_HOME/logs/dss-orchestrator/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###Install flow execution + installPackage + #echo "" + + SERVER_NAME=dss-workflow-server + SERVER_IP=$DSS_WORKFLOW_SERVER_INSTALL_IP + SERVER_PORT=$DSS_WORKFLOW_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-orchestrator/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-orchestrator + LOG_PATH=$SERVER_HOME/logs/dss-orchestrator/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install dss-workflow-server + installPackage + + ###install dss-data-api-server + SERVER_NAME=dss-data-api-server + SERVER_IP=$DSS_DATA_API_SERVER_INSTALL_IP + SERVER_PORT=$DSS_DATA_API_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-api/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-data-api + LOG_PATH=$SERVER_HOME/logs/dss-data-api/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + installPackage + + ###install dss-data-governance-server + SERVER_NAME=dss-data-governance-server + SERVER_IP=$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP + SERVER_PORT=$DSS_DATA_GOVERNANCE_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-governance/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-data-governance + LOG_PATH=$SERVER_HOME/logs/dss-data-governance/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + installPackage + + ###install dss-guide-server + SERVER_NAME=dss-guide-server + SERVER_IP=$DSS_GUIDE_SERVER_INSTALL_IP + SERVER_PORT=$DSS_GUIDE_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-guide/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-guide + LOG_PATH=$SERVER_HOME/logs/dss-guide/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + installPackage + + #echo "-----------------DSS install end--------------------" + #echo "" + +} +ENV_FLAG="dev" +installDssProject + +echo "Congratulations! You have installed DSS $DSS_VERSION successfully, please use sbin/dss-start-all.sh to start it!" + diff --git a/assembly/config/config.sh b/assembly/config/config.sh new file mode 100644 index 000000000..d36b2acb9 --- /dev/null +++ b/assembly/config/config.sh @@ -0,0 +1,87 @@ +### deploy user +deployUser=hadoop + +## max memory for services +SERVER_HEAP_SIZE="512M" + +### The install home path of DSS,Must provided +DSS_INSTALL_HOME=/appcom/Install/DSSInstall + +DSS_VERSION=1.1.0 + +DSS_FILE_NAME="dss-$DSS_VERSION" + +### Linkis EUREKA information. # Microservices Service Registration Discovery Center +EUREKA_INSTALL_IP=127.0.0.1 +EUREKA_PORT=20303 +### If EUREKA has safety verification, please fill in username and password +#EUREKA_USERNAME= +#EUREKA_PASSWORD= + +### Linkis Gateway information +GATEWAY_INSTALL_IP=127.0.0.1 +GATEWAY_PORT=9001 + +################### The install Configuration of all Micro-Services ##################### +# +# NOTICE: +# 1. If you just wanna try, the following micro-service configuration can be set without any settings. +# These services will be installed by default on this machine. +# 2. In order to get the most complete enterprise-level features, we strongly recommend that you install +# the following microservice parameters +# + +### DSS_SERVER +### This service is used to provide dss-server capability. + +### project-server +DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP=127.0.0.1 +DSS_FRAMEWORK_PROJECT_SERVER_PORT=9002 +### orchestrator-server +DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP=127.0.0.1 +DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT=9003 +### apiservice-server +DSS_APISERVICE_SERVER_INSTALL_IP=127.0.0.1 +DSS_APISERVICE_SERVER_PORT=9004 +### dss-workflow-server +DSS_WORKFLOW_SERVER_INSTALL_IP=127.0.0.1 +DSS_WORKFLOW_SERVER_PORT=9005 +### dss-flow-execution-server +DSS_FLOW_EXECUTION_SERVER_INSTALL_IP=127.0.0.1 +DSS_FLOW_EXECUTION_SERVER_PORT=9006 +###dss-scriptis-server +DSS_SCRIPTIS_SERVER_INSTALL_IP=127.0.0.1 +DSS_SCRIPTIS_SERVER_PORT=9008 + +###dss-data-api-server +DSS_DATA_API_SERVER_INSTALL_IP=127.0.0.1 +DSS_DATA_API_SERVER_PORT=9208 +###dss-data-governance-server +DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=127.0.0.1 +DSS_DATA_GOVERNANCE_SERVER_PORT=9209 +###dss-guide-server +DSS_GUIDE_SERVER_INSTALL_IP=127.0.0.1 +DSS_GUIDE_SERVER_PORT=9210 + +############## ############## dss_appconn_instance configuration start ############## ############## +####eventchecker表的地址,一般就是dss数据库 +EVENTCHECKER_JDBC_URL="jdbc:mysql://$MYSQL_HOST:$MYSQL_PORT/$MYSQL_DB?characterEncoding=UTF-8" +EVENTCHECKER_JDBC_USERNAME=$MYSQL_USER +EVENTCHECKER_JDBC_PASSWORD=$MYSQL_PASSWORD + +#### hive地址 +DATACHECKER_JOB_JDBC_URL="jdbc:mysql://127.0.0.1:3306/hive_gz_bdap_test_01?useUnicode=true" +DATACHECKER_JOB_JDBC_USERNAME=hadoop +DATACHECKER_JOB_JDBC_PASSWORD=hadoop +#### 元数据库,可配置成和DATACHECKER_JOB的一致 +DATACHECKER_BDP_JDBC_URL="jdbc:mysql://127.0.0.1:3306/uat2_metastore?characterEncoding=UTF-8" +DATACHECKER_BDP_JDBC_USERNAME=hadoop +DATACHECKER_BDP_JDBC_PASSWORD=hadoop + +EMAIL_HOST=smtp.163.com +EMAIL_PORT=25 +EMAIL_USERNAME=xxx@163.com +EMAIL_PASSWORD=xxxxx +EMAIL_PROTOCOL=smtp +############## ############## dss_appconn_instance configuration end ############## ############## + diff --git a/assembly/config/db.sh b/assembly/config/db.sh new file mode 100644 index 000000000..176f14419 --- /dev/null +++ b/assembly/config/db.sh @@ -0,0 +1,8 @@ +### for DSS-Server and Eventchecker APPCONN +MYSQL_HOST= +MYSQL_PORT= +MYSQL_DB= +MYSQL_USER= +MYSQL_PASSWORD= + + diff --git a/assembly/dss-package/pom.xml b/assembly/dss-package/pom.xml new file mode 100644 index 000000000..7b1402215 --- /dev/null +++ b/assembly/dss-package/pom.xml @@ -0,0 +1,186 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + dss-package + + + org.apache.linkis + linkis-rpc + ${linkis.version} + + + jackson-databind + com.fasterxml.jackson.core + + + jackson-core + com.fasterxml.jackson.core + + + jackson-core-asl + org.codehaus.jackson + + + httpclient + org.apache.httpcomponents + + + hibernate-validator + org.hibernate.validator + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + jackson-core + com.fasterxml.jackson.core + + + + + com.webank.wedatasphere.dss + dss-sender-service + ${dss.version} + + + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + + org.apache.httpcomponents + httpmime + ${httpmime.version} + + + commons-beanutils + commons-beanutils + ${beanutils.version} + + + + org.apache.linkis + linkis-hadoop-common + ${linkis.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + + org.apache.commons + commons-math3 + ${math3.version} + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + + + org.apache.linkis + linkis-storage + ${linkis.version} + + + + org.glassfish.jersey.ext + jersey-bean-validation + ${jersey.version} + + + + com.fasterxml + classmate + 1.3.4 + + + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + org.apache.maven.plugins + maven-antrun-plugin + + + package + + run + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + + + dist + package + + single + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + + + \ No newline at end of file diff --git a/assembly/dss-package/src/main/assembly/distribution.xml b/assembly/dss-package/src/main/assembly/distribution.xml new file mode 100644 index 000000000..09fce1e05 --- /dev/null +++ b/assembly/dss-package/src/main/assembly/distribution.xml @@ -0,0 +1,320 @@ + + + + dist + + dir + + true + dss-${dss.version} + + + + + + lib/dss-commons/ + true + true + false + true + true + + + + + + ${basedir}/../../ + . + + README* + LICENSE* + NOTICE* + + + + + + ${basedir}/../../conf + + conf + + **/* + + unix + + + + + ${basedir}/../../sbin + + sbin + + **/* + + unix + + + + + + ${basedir}/../../dss-framework/dss-framework-project-server/target/out/dss-framework-project-server/lib/ + + lib/dss-framework/dss-framework-project-server + + **/* + + + + + + + ${basedir}/../../dss-framework/framework-plugins/dss-framework-migrate-server/target/ + + lib/dss-framework/dss-framework-project-server + + *${dss.version}.jar + + + + + + + ${basedir}/../../dss-framework/dss-framework-orchestrator-server/target/out/dss-framework-orchestrator-server/lib/ + + lib/dss-framework/dss-framework-orchestrator-server/ + + **/* + + + + + + + ${basedir}/../../dss-apps/dss-apiservice-server/target/out/dss-apiservice-server/lib/ + + lib/dss-apps/dss-apiservice-server + + **/* + + + + + + + ${basedir}/../../dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/target/out/dss-workflow-server/lib/ + + lib/dss-orchestrator/dss-workflow-server/ + + **/* + + + + + + + ${basedir}/../../dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/target/out/dss-flow-execution-server/lib/ + + lib/dss-orchestrator/dss-flow-execution-server/ + + **/* + + + + + + + ${basedir}/../../dss-appconn/appconns/dss-datachecker-appconn/target/out/ + + dss-appconns + + **/* + + + + + + + ${basedir}/../../dss-appconn/appconns/dss-eventchecker-appconn/target/out/ + + dss-appconns + + **/* + + + + + + + ${basedir}/../../dss-appconn/appconns/dss-orchestrator-framework-appconn/target/out/ + + dss-appconns + + **/* + + + + + + + ${basedir}/../../dss-appconn/appconns/dss-workflow-appconn/target/out/ + + dss-appconns + + **/* + + + + + + + ${basedir}/../../dss-appconn/appconns/dss-schedulis-appconn/target/out/ + + dss-appconns + + **/* + + + + + + + ${basedir}/../../dss-appconn/appconns/dss-sso-appconn/target/out/ + + dss-appconns + + **/* + + + + + + + ${basedir}/../../dss-appconn/appconns/dss-scriptis-appconn/target/out/ + + dss-appconns + + **/* + + + + + + + + + + ${basedir}/../../dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/target/out/ + + dss-appconns + + **/* + + + + + + ${basedir}/../../dss-appconn/appconns/dss-dolphinscheduler-appconn/target/out/ + + dss-appconns + + **/* + + + + + + ${basedir}/../../dss-appconn/linkis-appconn-engineplugin/target/ + + dss-appconns + + *.zip + + + + + + ${basedir}/../../plugins/linkis/dss-gateway-support/target/ + + lib/dss-plugins/linkis/dss-gateway/ + + *${dss.version}.jar + + + + + + ${basedir}/../../plugins/azkaban/linkis-jobtype/target/ + + lib/dss-plugins/azkaban/linkis-jobtype/ + + *${dss.version}.zip + + + + + + + ${basedir}/../../dss-apps/dss-scriptis-server/target/out/dss-scriptis-server/lib/ + + lib/dss-apps/dss-scriptis-server + + **/* + + + + + + + ${basedir}/../../dss-data-api/dss-data-api-server/target/out/dss-data-api-server/lib/ + + lib/dss-data-api/dss-data-api-server + + **/* + + + + + + + ${basedir}/../../dss-data-governance/dss-data-governance-server/target/out/dss-data-governance-server/lib/ + + lib/dss-data-governance/dss-data-governance-server + + **/* + + + + + + + ${basedir}/../../dss-apps/dss-user-guide/dss-user-guide-server/target/out/dss-user-guide-server/lib/ + + lib/dss-guide/dss-guide-server + + **/* + + + + diff --git a/assembly/pom.xml b/assembly/pom.xml new file mode 100644 index 000000000..2e41028b0 --- /dev/null +++ b/assembly/pom.xml @@ -0,0 +1,74 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + pom + 4.0.0 + + dss-assembly + + dss-package + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + package + + run + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + + + dist + package + + single + + + false + wedatasphere-dss-${dss.version}-dist + false + false + + src/main/assembly/assembly.xml + + + + + + + + + \ No newline at end of file diff --git a/assembly/src/main/assembly/assembly.xml b/assembly/src/main/assembly/assembly.xml new file mode 100644 index 000000000..f7d796750 --- /dev/null +++ b/assembly/src/main/assembly/assembly.xml @@ -0,0 +1,83 @@ + + + + dist + + tar.gz + + false + + + + + + ${project.parent.basedir} + . + + README* + LICENSE* + NOTICE* + + + + + + + + ${project.parent.basedir}/assembly/config/ + + config + + **/* + + unix + + + + + ${project.parent.basedir}/assembly/bin/ + + bin + + **/* + + unix + + + + + ${project.parent.basedir}/db + + db + + **/* + + + + + + + ${project.parent.basedir}/assembly/dss-package/target/out + + + + **/* + + + + + diff --git a/conf/application-dss.yml b/conf/application-dss.yml new file mode 100644 index 000000000..b0473838d --- /dev/null +++ b/conf/application-dss.yml @@ -0,0 +1,23 @@ + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + #instance: + #prefer-ip-address: true + #instance-id: ${spring.cloud.client.ip-address}:${server.port} + #metadata-map: + #test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + diff --git a/conf/atlas-application.properties b/conf/atlas-application.properties new file mode 100644 index 000000000..e69de29bb diff --git a/conf/dss-apiservice-server.properties b/conf/dss-apiservice-server.properties new file mode 100644 index 000000000..5162f42bc --- /dev/null +++ b/conf/dss-apiservice-server.properties @@ -0,0 +1,42 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9206 +spring.spring.application.name=dss-apiservice-server + +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/apiservice/core/dao/mapper/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.apiservice.core.bo,com.webank.wedatasphere.dss.apiservice.core.vo +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.apiservice.core.dao + +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.apiservice.core.restful + +#sit +wds.linkis.server.version=v1 +wds.linkis.server.url= + +#test +wds.linkis.test.mode=false +wds.linkis.test.user= + + +#dsm +wds.linkis.server.dsm.admin.users= + + +#用于执行的datasource配置 +wds.linkis.datasource.hikari.maximumPoolSize=100 +wds.linkis.datasource.hikari.minimumIdle=10 diff --git a/conf/dss-data-api-server.properties b/conf/dss-data-api-server.properties new file mode 100644 index 000000000..fe386097e --- /dev/null +++ b/conf/dss-data-api-server.properties @@ -0,0 +1,39 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9208 +spring.spring.application.name=dss-data-api-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.api.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/api/server/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.api.server.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.api.server.dao + +#wds.linkis.gateway.ip=127.0.0.1 +#wds.linkis.gateway.port=9001 +#wds.linkis.gateway.url=http://127.0.0.1:9001/ diff --git a/conf/dss-data-governance-server.properties b/conf/dss-data-governance-server.properties new file mode 100644 index 000000000..b3980c6cc --- /dev/null +++ b/conf/dss-data-governance-server.properties @@ -0,0 +1,53 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9209 +spring.spring.application.name=dss-data-governance-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.asset.restful,com.webank.wedatasphere.dss.data.classification.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/asset/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.asset.entity +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.asset.dao,com.webank.wedatasphere.dss.data.warehouse.dao,com.webank.wedatasphere.dss.data.warehouse.mapper + +#wds.linkis.gateway.ip=127.0.0.1 +#wds.linkis.gateway.port=9001 +#wds.linkis.gateway.url=http://127.0.0.1:9001/ + + +# atlas config +atlas.rest.address=http://xxxxxxx:21000 +atlas.username=xxxxxxxx +atlas.password=yyyyyyyyy +atlas.client.readTimeoutMSecs=60000 +atlas.client.connectTimeoutMSecs=60000 + +atlas.cluster.name=primary + +# hive metadata config +metastore.datasource.driver=com.mysql.jdbc.Driver +metastore.datasource.url=jdbc:mysql://xxxxxx:yyyy/metastore?characterEncoding=UTF-8 +metastore.datasource.username=xxxxxx +metastore.datasource.password=yyyyyy \ No newline at end of file diff --git a/conf/dss-flow-execution-server.properties b/conf/dss-flow-execution-server.properties new file mode 100644 index 000000000..1d8eb8d28 --- /dev/null +++ b/conf/dss-flow-execution-server.properties @@ -0,0 +1,55 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9006 + +spring.spring.application.name=dss-flow-entrance + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/*.xml,classpath*:org/apache/linkis/jobhistory/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage= + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.flow.execution.entrance.dao,org.apache.linkis.jobhistory.dao + + +wds.linkis.server.restful.scan.packages=org.apache.linkis.entrance.restful,com.webank.wedatasphere.dss.flow.execution.entrance.restful + +#wds.linkis.server.component.exclude.classes=org.apache.linkis.DataWorkCloudApplication + +wds.linkis.engine.application.name=flowExecutionEngine +wds.linkis.enginemanager.application.name=flowExecution + +wds.linkis.query.application.name=linkis-ps-publicservice + +wds.linkis.console.config.application.name=linkis-ps-publicservice +wds.linkis.engine.creation.wait.time.max=20m +wds.linkis.server.version=v1 + +wds.linkis.server.socket.mode=true + +wds.linkis.client.flow.adminuser=ws +wds.linkis.client.flow.author.user.token=WS-AUTH + +wds.linkis.server.component.exclude.classes=org.apache.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,org.apache.linkis.entranceclient.conf.ClientSpringConfiguration + +wds.linkis.server.component.exclude.packages=org.apache.linkis.entrance.restful. +spring.spring.main.allow-bean-definition-overriding=true + +wds.linkis.entrance.config.log.path=file:///appcom/tmp/dss/ +wds.linkis.spark.engine.version=2.4.3 +wds.linkis.hive.engine.version=2.3.3 diff --git a/conf/dss-framework-orchestrator-server.properties b/conf/dss-framework-orchestrator-server.properties new file mode 100644 index 000000000..a2b589a16 --- /dev/null +++ b/conf/dss-framework-orchestrator-server.properties @@ -0,0 +1,41 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9003 +spring.spring.application.name=dss-framework-orchestrator-server-dev + +wds.linkis.test.mode=false + +wds.linkis.test.user=hadoop + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.orchestrator.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/orchestrator/core/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/server/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/application/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/workspace/mapper/impl/*.xml,classpath*:com/webank/wedatasphere/dss/workspace/common/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/orchestrator/db/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.server.entity,com.webank.wedatasphere.dss.application.entity,com.webank.wedatasphere.dss.framework.appconn.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.orchestrator.core.dao,com.webank.wedatasphere.dss.server.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.workspace.mapper,com.webank.wedatasphere.dss.workspace.common.dao,com.webank.wedatasphere.dss.workspace.common.dao,com.webank.wedatasphere.dss.orchestrator.db.dao + + +##export file dir +wds.dss.server.export.url=/appcom/tmp/dss diff --git a/conf/dss-framework-project-server.properties b/conf/dss-framework-project-server.properties new file mode 100644 index 000000000..6b8086de8 --- /dev/null +++ b/conf/dss-framework-project-server.properties @@ -0,0 +1,37 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9202 +spring.spring.application.name=dss-framework-project-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.framework.workspace.restful,com.webank.wedatasphere.dss.framework.project.restful,com.webank.wedatasphere.dss.framework.release.restful,com.webank.wedatasphere.dss.framework.appconn.restful,com.webank.wedatasphere.dss.framework.admin.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/workspace/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/application/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/project/dao/impl/*Mapper.xml,classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/release/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/admin/xml/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.application.entity,com.webank.wedatasphere.dss.common.entity,com.webank.wedatasphere.dss.framework.workspace.bean,com.webank.wedatasphere.dss.framework.project.entity,com.webank.wedatasphere.dss.framework.appconn.entity,com.webank.wedatasphere.dss.framework.release.entity,com.webank.wedatasphere.dss.framework.admin.pojo.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.framework.workspace.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.framework.project.dao,com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.framework.release.dao,com.webank.wedatasphere.dss.framework.admin.xml + +wds.dss.appconn.checker.development.ignore.list=workflow,sendemail +wds.dss.appconn.checker.project.ignore.list=visualis + diff --git a/conf/dss-guide-server.properties b/conf/dss-guide-server.properties new file mode 100644 index 000000000..3a2f488d7 --- /dev/null +++ b/conf/dss-guide-server.properties @@ -0,0 +1,55 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9210 +spring.spring.application.name=dss-guide-server + +spring.jackson.date-format=yyyy-MM-dd HH:mm:ss +spring.jackson.time-zone=GMT+8 + +wds.linkis.server.version=v1 + +wds.linkis.log.clear=true + + + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.guide.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/guide/server/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.guide.server.entity +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.guide.server.dao + + +## guide_images_path +guide.content.images.path=/opt/dss/dss-guide-server/guide_images/ +guide.chapter.images.path=/opt/dss/dss-guide-server/guide_images/ + +#gitbook +#The machine where the file exists +target.ip.address=127.0.0.1 +#The file path of the machine where the file is stored +host.gitbook.path=/appcom/Install/ApacheInstall/gitbook_books +#The path to copy the file to the current machine +target.gitbook.path=/appcom/Install/ApacheInstall +#SUMMARY.md ignore absolve +summary.ignore.model=km +#sync model include: gitbook or database +guide.sync.model=gitbook diff --git a/conf/dss-scriptis-server.properties b/conf/dss-scriptis-server.properties new file mode 100644 index 000000000..cda41694b --- /dev/null +++ b/conf/dss-scriptis-server.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9009 +spring.spring.application.name=dss-scriptis-server + +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/scriptis/dao/mapper/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.scriptis.vo +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.scriptis.dao + +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.scriptis.restful + +#sit +wds.linkis.server.version=v1 +wds.linkis.server.url= + +#test +wds.linkis.test.mode=false +wds.linkis.test.user= + +wds.dss.scriptis.global.limits.exportResEnable=false +wds.dss.scriptis.global.limits.exportTableEnable=false \ No newline at end of file diff --git a/conf/dss-workflow-server.properties b/conf/dss-workflow-server.properties new file mode 100644 index 000000000..df4bd6dea --- /dev/null +++ b/conf/dss-workflow-server.properties @@ -0,0 +1,44 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9207 +spring.spring.application.name=dss-workflow-server-dev + +wds.linkis.test.mode=false + +wds.linkis.test.user= + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.workflow.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.workflow.entity,com.webank.wedatasphere.dss.framework.appconn.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.workflow.dao,com.webank.wedatasphere.dss.framework.appconn.dao + +##import file dir +wds.dss.file.upload.dir=/appcom/tmp/uploads + +wds.dss.server.export.env=DEV +wds.dss.server.import.env=DEV + diff --git a/conf/dss.properties b/conf/dss.properties new file mode 100644 index 000000000..b68aa7ff6 --- /dev/null +++ b/conf/dss.properties @@ -0,0 +1,37 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +wds.linkis.gateway.ip=127.0.0.1 +wds.linkis.gateway.port=9001 +wds.linkis.gateway.url=http://127.0.0.1:9001/ +wds.linkis.gateway.wtss.url=http://127.0.0.1:9001/ + +wds.linkis.mysql.is.encrypt=false +wds.linkis.server.mybatis.datasource.url= +wds.linkis.server.mybatis.datasource.username= +wds.linkis.server.mybatis.datasource.password= + +wds.dss.esb.appid= +wds.dss.esb.token= + +wds.dss.appconn.scheduler.job.label=dev + + +wds.linkis.reflect.scan.package=org.apache.linkis,com.webank.wedatasphere.dss +spring.spring.mvc.servlet.path=/api/rest_j/v1 +spring.spring.servlet.multipart.max-file-size=200MB +spring.spring.servlet.multipart.max-request-size=200MB +wds.dss.project.strict.mode=true diff --git a/conf/log4j.properties b/conf/log4j.properties new file mode 100644 index 000000000..c94a1c841 --- /dev/null +++ b/conf/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/dss-apiservice-server.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/conf/log4j2.xml b/conf/log4j2.xml new file mode 100644 index 000000000..f3d6e66bc --- /dev/null +++ b/conf/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/token.properties b/conf/token.properties new file mode 100644 index 000000000..5674d0245 --- /dev/null +++ b/conf/token.properties @@ -0,0 +1,17 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +${userName}=${password} \ No newline at end of file diff --git a/db/apps/dss_apiservice_ddl.sql b/db/apps/dss_apiservice_ddl.sql new file mode 100644 index 000000000..cd75b8258 --- /dev/null +++ b/db/apps/dss_apiservice_ddl.sql @@ -0,0 +1,107 @@ +DROP TABLE IF EXISTS `dss_apiservice_api`; +CREATE TABLE `dss_apiservice_api` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `name` varchar(180) NOT NULL COMMENT '服务名称', + `alias_name` varchar(200) NOT NULL COMMENT '服务中文名称', + `path` varchar(180) NOT NULL COMMENT '服务路径', + `protocol` int(11) NOT NULL COMMENT '协议: http, https', + `method` varchar(10) NOT NULL COMMENT '方法: post, put, delete', + `tag` varchar(200) DEFAULT NULL COMMENT '标签', + `scope` varchar(50) DEFAULT NULL COMMENT '范围', + `description` varchar(200) DEFAULT NULL COMMENT '服务描述', + `status` int(11) DEFAULT '0' COMMENT '服务状态,默认0是停止,1是运行中,2是删除', + `type` varchar(50) DEFAULT NULL COMMENT '服务引擎类型', + `run_type` varchar(50) DEFAULT NULL COMMENT '脚本类型', + `create_time` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `modify_time` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + `creator` varchar(50) DEFAULT NULL COMMENT '创建者', + `modifier` varchar(50) DEFAULT NULL COMMENT '修改者', + `script_path` varchar(180) NOT NULL COMMENT '脚本路径', + `workspaceID` int(11) NOT NULL COMMENT '工作空间ID', + `api_comment` varchar(1024) DEFAULT NULL COMMENT '服务备注', + PRIMARY KEY (`id`), + UNIQUE KEY `idx_uniq_config_name` (`name`), + UNIQUE KEY `idx_uniq_dconfig_path` (`path`), + KEY `idx_dss_script_path` (`script_path`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='服务api配置表'; + +DROP TABLE IF EXISTS `dss_apiservice_param`; +CREATE TABLE `dss_apiservice_param` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_version_id` bigint(20) NOT NULL COMMENT '服务api版本id', + `name` varchar(200) NOT NULL COMMENT '名称', + `display_name` varchar(50) DEFAULT NULL COMMENT '展示名', + `type` varchar(50) DEFAULT NULL COMMENT '类型', + `required` tinyint(1) DEFAULT '1' COMMENT '是否必须: 0否, 1是', + `default_value` varchar(1024) DEFAULT NULL COMMENT '参数的默认值', + `description` varchar(200) DEFAULT NULL COMMENT '描述', + `details` varchar(1024) DEFAULT NULL COMMENT '变量的详细说明', + PRIMARY KEY (`id`), + KEY `idx_api_version_id` (`api_version_id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='apiservice 参数表'; + +DROP TABLE IF EXISTS `dss_apiservice_api_version`; +CREATE TABLE `dss_apiservice_api_version` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_id` bigint(20) NOT NULL COMMENT '服务的ID', + `version` varchar(50) NOT NULL COMMENT '服务对应的版本信息', + `bml_resource_id` varchar(50) NOT NULL COMMENT 'bml资源id', + `bml_version` varchar(20) NOT NULL COMMENT 'bml版本', + `source` varchar(200) DEFAULT NULL COMMENT '来源', + `creator` varchar(50) DEFAULT NULL COMMENT '创建者', + `create_time`timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `status` tinyint(1) default '1' COMMENT '0表示被禁用,1表示正在运行', + `metadata_info` varchar(5000) COMMENT '发布者库表信息', + `auth_id` varchar(200) COMMENT '用于与datamap交互的UUID', + `datamap_order_no` varchar(200) DEFAULT NULL COMMENT 'datamap审批单号码', + PRIMARY KEY(`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='服务api版本表'; + +DROP TABLE IF EXISTS `dss_apiservice_token_manager`; +CREATE TABLE `dss_apiservice_token_manager` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_version_id` bigint(20) NOT NULL COMMENT '服务api版本id', + `api_id` bigint(20) NOT NULL COMMENT '服务api配置id', + `publisher` varchar(20) NOT NULL COMMENT '发布用户', + `user` varchar(20) NOT NULL COMMENT '申请用户', + `apply_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '申请时间', + `duration` int(10) NOT NULL COMMENT '时长', + `reason` varchar(200) DEFAULT NULL COMMENT '申请原因', + `ip_whitelist` varchar(200) DEFAULT NULL COMMENT 'IP白名单', + `status` tinyint(1) DEFAULT '1' COMMENT '状态0过期,1有效期内', + `caller` varchar(50) DEFAULT NULL COMMENT '调用方', + `access_limit` varchar(50) DEFAULT NULL COMMENT '限流情况', + `apply_source` varchar(200) DEFAULT NULL COMMENT '申请来源', + `token` varchar(500) DEFAULT NULL COMMENT 'token内容', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='apiservice token管理表'; + +DROP TABLE IF EXISTS `dss_apiservice_approval`; +CREATE TABLE `dss_apiservice_approval` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_id` bigint(20) NOT NULL COMMENT '服务api id', + `api_version_id` bigint(20) NOT NULL COMMENT '版本id', + `approval_name` varchar(50) NOT NULL COMMENT '审批单名称', + `apply_user` varchar(1024) NOT NULL COMMENT '申请用户', + `execute_user` varchar(50) DEFAULT NULL COMMENT '代理执行用户,用,分割', + `creator` varchar(50) NOT NULL COMMENT '创建者', + `status` int(10) DEFAULT '0' COMMENT '申请状态,提单成功1,审批中2,成功3,失败4', + `create_time` timestamp NOT null DEFAULT CURRENT_TIMESTAMP COMMENT '审批单创建时间', + `update_time` timestamp NOT null DEFAULT CURRENT_TIMESTAMP COMMENT '审批单状态更新时间', + `approval_no` varchar(500) NOT NULL COMMENT '审批单号', + PRIMARY KEY(`id`), + UNIQUE KEY `idx_uniq_api_version_id` (`api_version_id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='apiservice 审批单表'; + +DROP TABLE IF EXISTS `dss_apiservice_access_info`; +CREATE TABLE `dss_apiservice_access_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_id` bigint(20) NOT NULL COMMENT '服务id', + `api_version_id` bigint(20) NOT NULL COMMENT '版本id', + `api_name` varchar(50) NOT NULL COMMENT '服务名称', + `login_user` varchar(50) NOT NULL COMMENT '提交用户', + `execute_user` varchar(50) DEFAULT NULL COMMENT '代理执行用户', + `api_publisher` varchar(50) NOT NULL COMMENT 'api创建者', + `access_time` timestamp NOT null DEFAULT CURRENT_TIMESTAMP COMMENT '访问时间', + PRIMARY KEY(`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='apiservice 访问信息表'; \ No newline at end of file diff --git a/db/apps/dss_dataapi_ddl.sql b/db/apps/dss_dataapi_ddl.sql new file mode 100644 index 000000000..84cc55adc --- /dev/null +++ b/db/apps/dss_dataapi_ddl.sql @@ -0,0 +1,96 @@ +DROP TABLE IF EXISTS `dss_dataapi_config`; +CREATE TABLE `dss_dataapi_config` ( + `id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT COMMENT '主键', + `workspace_id` BIGINT ( 20 ) NOT NULL COMMENT '工作空间id', + `api_name` VARCHAR ( 255 ) NOT NULL COMMENT 'API名称', + `api_path` VARCHAR ( 255 ) NOT NULL unique COMMENT 'API Path', + `group_id` BIGINT ( 20 ) NOT NULL COMMENT 'API组id', + `api_type` VARCHAR ( 20 ) NOT NULL COMMENT 'API类型:GUIDE-向导模式,SQL-脚本模式', + `protocol` VARCHAR ( 20 ) NOT NULL COMMENT 'Http协议', + `datasource_id` BIGINT ( 20 ) NOT NULL COMMENT '数据源id', + `datasource_name` VARCHAR ( 50 ) DEFAULT NULL COMMENT '数据源名称', + `datasource_type` VARCHAR ( 20 ) DEFAULT NULL COMMENT '数据源类型', + + `sql` text COMMENT 'sql模板', + `tbl_name` VARCHAR ( 100 ) DEFAULT NULL COMMENT '数据表名称', + `req_fields` VARCHAR ( 1000 ) DEFAULT NULL COMMENT '请求字段名称', + `res_fields` VARCHAR ( 1000 ) DEFAULT NULL COMMENT '返回字段名称', + `order_fields` VARCHAR ( 500 ) DEFAULT NULL COMMENT '排序字段名称及方式', + `is_test` TINYINT ( 1 ) DEFAULT '0' COMMENT '是否测试成功:0未测试(默认),1测试成功', + `status` TINYINT ( 1 ) DEFAULT '0' COMMENT 'API状态:0未发布(默认),1已发布', + `previlege` VARCHAR ( 20 ) DEFAULT 'WORKSPACE' COMMENT 'WORKSPACE,PRIVATE', + `method` VARCHAR ( 20 ) DEFAULT NULL COMMENT 'HTTPS,HTTP', + `describe` VARCHAR ( 255 ) DEFAULT NULL COMMENT '描述', + `memory` INT DEFAULT NULL COMMENT '内存大小', + `req_timeout` INT DEFAULT NULL COMMENT '请求超时时间', + `label` VARCHAR ( 255 ) DEFAULT NULL COMMENT '标签', + `create_by` VARCHAR ( 255 ) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` VARCHAR ( 255 ) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `is_delete` TINYINT ( 1 ) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + `page_size` int DEFAULT 0 COMMENT '每页数据大小', + + PRIMARY KEY ( `id` ) +) ENGINE = INNODB DEFAULT CHARSET = utf8 COMMENT = 'API'; + +DROP TABLE IF EXISTS `dss_dataapi_group`; +CREATE TABLE `dss_dataapi_group` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL COMMENT '工作空间id', + `name` varchar(255) NOT NULL COMMENT '名称', + `note` varchar(255) DEFAULT NULL COMMENT '描述', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE = INNODB DEFAULT CHARSET = utf8 COMMENT='服务组'; + +DROP TABLE IF EXISTS `dss_dataapi_auth`; +CREATE TABLE `dss_dataapi_auth` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) NOT NULL COMMENT '工作空间ID', + `caller` varchar(255) DEFAULT NULL COMMENT '调用者名称', + `token` varchar(255) DEFAULT NULL COMMENT 'token字符串', + `expire` datetime DEFAULT NULL COMMENT 'token过期时间', + `group_id` bigint(20) DEFAULT NULL COMMENT 'api组', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='API认证'; + +DROP TABLE IF EXISTS `dss_dataapi_call`; +CREATE TABLE `dss_dataapi_call` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `api_id` bigint(11) NOT NULL COMMENT 'API ID', + `params_value` text COMMENT '调用参数名称和值', + `status` tinyint(255) DEFAULT NULL COMMENT '执行状态:1成功,2失败,3超时', + `time_start` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '开始时间', + `time_end` datetime DEFAULT NULL COMMENT '结束时间', + `time_length` bigint(20) DEFAULT NULL COMMENT '调用时长', + `caller` varchar(255) DEFAULT NULL COMMENT '调用者名称', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='API调用记录' +; +DROP TABLE IF EXISTS `dss_dataapi_datasource`; +CREATE TABLE `dss_dataapi_datasource` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL COMMENT '工作空间id', + `name` varchar(255) NOT NULL COMMENT '名称', + `note` varchar(255) DEFAULT NULL COMMENT '描述', + `type` varchar(255) NOT NULL COMMENT '数据库类型', + `url` varchar(255) NOT NULL COMMENT '连接url', + `username` varchar(255) NOT NULL COMMENT '用户名', + `pwd` varchar(255) NOT NULL COMMENT '密码', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='数据源'; \ No newline at end of file diff --git a/db/apps/dss_guide_ddl.sql b/db/apps/dss_guide_ddl.sql new file mode 100644 index 000000000..f0a072941 --- /dev/null +++ b/db/apps/dss_guide_ddl.sql @@ -0,0 +1,62 @@ +DROP TABLE IF EXISTS `dss_guide_group`; +CREATE TABLE IF NOT EXISTS `dss_guide_group` ( + `id` BIGINT(13) NOT NULL AUTO_INCREMENT, + `path` VARCHAR(100) NOT NULL COMMENT '页面URL路径', + `title` VARCHAR(255) DEFAULT NULL COMMENT '标题', + `description` VARCHAR(200) DEFAULT NULL COMMENT '描述', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='用户向导页面'; + +DROP TABLE IF EXISTS `dss_guide_content`; +CREATE TABLE IF NOT EXISTS `dss_guide_content` ( + `id` BIGINT(13) NOT NULL AUTO_INCREMENT, + `group_id` BIGINT(50) NOT NULL COMMENT '所属页面ID', + `path` VARCHAR(100) NOT NULL COMMENT '所属页面URL路径', + `title` VARCHAR(255) DEFAULT NULL COMMENT '标题', + `title_alias` VARCHAR(50) DEFAULT NULL COMMENT '标题简称', + `seq` VARCHAR(20) DEFAULT NULL COMMENT '序号', + `type` INT(1) DEFAULT '1' COMMENT '类型: 1-步骤step,2-问题question', + `content` TEXT DEFAULT NULL COMMENT 'Markdown格式的内容', + `content_html` MEDIUMTEXT DEFAULT NULL COMMENT 'Markdown内容转化为HTML格式', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='用户向导页面内容详情'; + +DROP TABLE IF EXISTS `dss_guide_catalog`; +CREATE TABLE IF NOT EXISTS `dss_guide_catalog` ( + `id` BIGINT(13) NOT NULL AUTO_INCREMENT, + `parent_id` BIGINT(13) NOT NULL COMMENT '父级目录ID,-1代表最顶级目录', + `title` VARCHAR(255) DEFAULT NULL COMMENT '标题', + `description` VARCHAR(200) DEFAULT NULL COMMENT '描述', + `create_by` VARCHAR(255) DEFAULT NULL COMMENT '创建者', + `create_time` DATETIME DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` VARCHAR(255) DEFAULT NULL COMMENT '更新者', + `update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` TINYINT(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='用户向导知识库目录'; + +DROP TABLE IF EXISTS `dss_guide_chapter`; +CREATE TABLE IF NOT EXISTS `dss_guide_chapter` ( + `id` BIGINT(13) NOT NULL AUTO_INCREMENT, + `catalog_id` BIGINT(13) NOT NULL COMMENT '目录ID', + `title` VARCHAR(255) DEFAULT NULL COMMENT '标题', + `title_alias` VARCHAR(50) DEFAULT NULL COMMENT '标题简称', + `content` TEXT DEFAULT NULL COMMENT 'Markdown格式的内容', + `content_html` MEDIUMTEXT DEFAULT NULL COMMENT 'Markdown转换为html内容', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='用户向导知识库文章'; \ No newline at end of file diff --git a/db/dss_ddl.sql b/db/dss_ddl.sql new file mode 100644 index 000000000..938961d09 --- /dev/null +++ b/db/dss_ddl.sql @@ -0,0 +1,592 @@ +DROP TABLE IF EXISTS `dss_appconn`; +CREATE TABLE `dss_appconn` ( + `id` int(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `appconn_name` varchar(64) UNIQUE NOT NULL COMMENT 'appconn的名称', + `is_user_need_init` tinyint(1) DEFAULT NULL COMMENT '是否需要初始化', + `level` int(8) DEFAULT NULL COMMENT '等级', + `if_iframe` tinyint(1) DEFAULT NULL COMMENT '是否能iframe嵌入', + `is_external` tinyint(1) DEFAULT NULL COMMENT '是否是外部接入应用', + `reference` varchar(255) DEFAULT NULL COMMENT '需要关联的某一个AppConn标识', + `class_name` varchar(255) DEFAULT NULL COMMENT '需要关联的某一个AppConn标识', + `appconn_class_path` varchar(255) DEFAULT NULL COMMENT '需要关联的某一个AppConn标识', + `resource` varchar(255) DEFAULT NULL COMMENT 'bml的资源ID', + PRIMARY KEY (`id`), + UNIQUE KEY `idx_appconn_name` (`appconn_name`) +) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='dss appconn表'; + +DROP TABLE IF EXISTS `dss_appconn_instance`; +CREATE TABLE `dss_appconn_instance` ( + `id` int(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `appconn_id` int(20) NOT NULL COMMENT 'appconn的主键', + `label` varchar(128) NOT NULL COMMENT '实例的标签', + `url` varchar(128) DEFAULT NULL COMMENT '访问第三方的url', + `enhance_json` varchar(1024) DEFAULT NULL COMMENT 'json格式的配置', + `homepage_uri` varchar(255) DEFAULT NULL COMMENT '主页uri,非URL', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='dss instance的实例表'; + +/* +--------------------------------------------------------------------- +------------------- DSS Orchestrator Framework --------------------- +--------------------------------------------------------------------- +*/ + +DROP TABLE IF EXISTS `dss_orchestrator_info`; +CREATE TABLE `dss_orchestrator_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL, + `type` varchar(255) NOT NULL, + `desc` varchar(1024) DEFAULT NULL, + `creator` varchar(100) NOT NULL, + `create_time` datetime DEFAULT NULL, + `project_id` bigint(20) DEFAULT NULL, + `uses` varchar(500) DEFAULT NULL, + `appconn_name` varchar(1024) NOT NULL, + `uuid` varchar(180) NOT NULL, + `secondary_type` varchar(500) DEFAULT NULL, + `is_published` tinyint(1) NOT NULL DEFAULT '0', + `workspace_id` int(11) DEFAULT NULL COMMENT '空间id', + `orchestrator_mode` varchar(100) DEFAULT NULL COMMENT '编排模式,取得的值是dss_dictionary中的dic_key(parent_key=p_arrangement_mode)', + `orchestrator_way` varchar(256) DEFAULT NULL COMMENT '编排方式', + `orchestrator_level` varchar(32) DEFAULT NULL COMMENT '工作流级别', + `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `unique_idx_uuid` (`uuid`) + ) ENGINE=InnoDB AUTO_INCREMENT=326 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + + +DROP TABLE IF EXISTS `dss_orchestrator_version_info`; +CREATE TABLE `dss_orchestrator_version_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `orchestrator_id` bigint(20) NOT NULL, + `app_id` bigint(20) DEFAULT NULL, + `source` varchar(255) DEFAULT NULL, + `version` varchar(255) DEFAULT NULL, + `comment` varchar(255) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `updater` varchar(32) DEFAULT NULL, + `project_id` bigint(20) DEFAULT NULL, + `content` varchar(255) DEFAULT NULL, + `context_id` varchar(200) DEFAULT NULL COMMENT '上下文ID', + `valid_flag` INT(1) DEFAULT '1' COMMENT '版本有效标示,0:无效;1:有效', + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=422 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + +DROP TABLE IF EXISTS `dss_orchestrator_ref_orchestration_relation`; +CREATE TABLE `dss_orchestrator_ref_orchestration_relation` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `orchestrator_id` bigint(20) NOT NULL COMMENT 'dss的编排模式id', + `ref_project_id` bigint(20) DEFAULT NULL COMMENT '调度系统关联的工程Id', + `ref_orchestration_id` int(11) DEFAULT NULL COMMENT '调度系统工作流的id(调用SchedulerAppConn的OrchestrationOperation服务返回的orchestrationId)', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=326 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + +/* +--------------------------------------------------------------------- +------------------- DSS Project Framework --------------------- +--------------------------------------------------------------------- +*/ + +DROP TABLE IF EXISTS `dss_project`; +CREATE TABLE `dss_project` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(200) COLLATE utf8_bin DEFAULT NULL, + `source` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT 'Source of the dss_project', + `description` text COLLATE utf8_bin, + `user_id` bigint(20) DEFAULT NULL, + `username` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `workspace_id` int(11) NOT NULL DEFAULT '0', + `create_time` datetime DEFAULT NULL, + `create_by` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '创建人', + `update_time` datetime DEFAULT NULL, + `update_by` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '修改人', + `org_id` bigint(20) DEFAULT NULL COMMENT 'Organization ID', + `visibility` bit(1) DEFAULT NULL, + `is_transfer` bit(1) DEFAULT NULL COMMENT 'Reserved word', + `initial_org_id` bigint(20) DEFAULT NULL, + `isArchive` bit(1) DEFAULT b'0' COMMENT 'If it is archived', + `pic` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `star_num` int(11) DEFAULT '0', + `product` varchar(200) COLLATE utf8_bin DEFAULT NULL, + `application_area` tinyint(1) DEFAULT NULL, + `business` varchar(200) COLLATE utf8_bin DEFAULT NULL, + `is_personal` tinyint(4) NOT NULL DEFAULT '0', + `create_by_str` varchar(256) COLLATE utf8_bin DEFAULT NULL, + `update_by_str` varchar(256) COLLATE utf8_bin DEFAULT NULL, + `dev_process` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT '开发流程,多个以英文逗号分隔,取得的值是dss_workspace_dictionary中的dic_key(parent_key=p_develop_process)', + `orchestrator_mode` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT '编排模式,多个以英文逗号分隔,取得的值是dss_workspace_dictionary中的dic_key(parent_key=p_arrangement_mode或下面一级)', + `visible` tinyint(4) DEFAULT '1' COMMENT '0:已删除;1:未删除(默认)', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=313 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; + + +DROP TABLE IF EXISTS `dss_project_user`; +CREATE TABLE `dss_project_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `project_id` int(10) NOT NULL, + `username` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `workspace_id` bigint(20) DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1859 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; + +DROP TABLE IF EXISTS `dss_appconn_project_relation`; +CREATE TABLE `dss_appconn_project_relation` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `project_id` bigint(20) NOT NULL, + `appconn_instance_id` bigint(20) NOT NULL, + `appconn_instance_project_id` bigint(20) NOT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=90 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + +/* +--------------------------------------------------------------------- +--------------------- DSS Workspace Framework ---------------------- +--------------------------------------------------------------------- +*/ + +DROP TABLE IF EXISTS `dss_workspace`; +CREATE TABLE `dss_workspace` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `label` varchar(255) DEFAULT NULL, + `description` varchar(255) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `department` varchar(255) DEFAULT NULL, + `product` varchar(255) DEFAULT NULL, + `source` varchar(255) DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL COMMENT '最新修改用户', + `workspace_type` varchar(20) DEFAULT NULL comment '工作空间类型', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB AUTO_INCREMENT=224 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workspace_homepage`; +CREATE TABLE `dss_workspace_homepage` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `workspace_id` int(10) NOT NULL, + `role_id` int(20) DEFAULT NULL, + `homepage_url` varchar(256) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1213 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workspace_dictionary`; +CREATE TABLE `dss_workspace_dictionary` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `workspace_id` int(11) DEFAULT '0' COMMENT '空间ID,默认为0,所有空间都有', + `parent_key` varchar(200) DEFAULT '0' COMMENT '父key', + `dic_name` varchar(200) NOT NULL COMMENT '名称', + `dic_name_en` varchar(300) DEFAULT NULL COMMENT '名称(英文)', + `dic_key` varchar(200) NOT NULL COMMENT 'key 相当于编码,空间是w_开头,工程是p_', + `dic_value` varchar(500) DEFAULT NULL COMMENT 'key对应的值', + `dic_value_en` varchar(1000) DEFAULT NULL COMMENT 'key对应的值(英文)', + `title` varchar(200) DEFAULT NULL COMMENT '标题', + `title_en` varchar(400) DEFAULT NULL COMMENT '标题(英文)', + `url` varchar(200) DEFAULT NULL COMMENT 'url', + `url_type` int(1) DEFAULT '0' COMMENT 'url类型: 0-内部系统,1-外部系统;默认是内部', + `icon` varchar(200) DEFAULT NULL COMMENT '图标', + `order_num` int(2) DEFAULT '1' COMMENT '序号', + `remark` varchar(1000) DEFAULT NULL COMMENT '备注', + `create_user` varchar(100) DEFAULT NULL COMMENT '创建人', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `idx_unique_workspace_id` (`workspace_id`,`dic_key`), + KEY `idx_parent_key` (`parent_key`), + KEY `idx_dic_key` (`dic_key`) +) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8 COMMENT='数据字典表'; + +DROP TABLE IF EXISTS `dss_workspace_role`; +CREATE TABLE `dss_workspace_role` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `workspace_id` varchar(255) DEFAULT NULL, + `name` varchar(255) DEFAULT NULL, + `front_name` varchar(255) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `description` varchar(512) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `workspace_id` (`workspace_id`,`name`) +) ENGINE=InnoDB AUTO_INCREMENT=10 DEFAULT CHARSET=utf8 CHECKSUM=1 DELAY_KEY_WRITE=1 ROW_FORMAT=DYNAMIC; + + + +DROP TABLE IF EXISTS `dss_user`; +CREATE TABLE `dss_user` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `username` varchar(64) DEFAULT NULL, + `name` varchar(64) DEFAULT NULL, + `is_first_login` tinyint(1) DEFAULT NULL, + `dept_id` int(11) DEFAULT NULL, + `is_admin` tinyint(1) DEFAULT '0' COMMENT '是否管理员(1:是;0:否)', + `email` varchar(50) DEFAULT '' COMMENT '用户邮箱', + `phonenumber` varchar(11) DEFAULT '' COMMENT '手机号码', + `password` varchar(100) DEFAULT '' COMMENT '密码', + `del_flag` char(1) DEFAULT '0' COMMENT '删除标志(0代表存在 2代表删除)', + `create_by` varchar(64) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `remark` varchar(500) DEFAULT NULL COMMENT '备注', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=214 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_workspace_admin_dept`; +CREATE TABLE `dss_workspace_admin_dept` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '部门id', + `parent_id` bigint(20) DEFAULT '0' COMMENT '父部门id', + `ancestors` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '祖级列表', + `dept_name` varchar(30) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '部门名称', + `order_num` int(4) DEFAULT '0' COMMENT '显示顺序', + `leader` varchar(20) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '负责人', + `phone` varchar(11) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '联系电话', + `email` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '邮箱', + `status` char(1) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '0' COMMENT '部门状态(0正常 1停用)', + `del_flag` char(1) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '0' COMMENT '删除标志(0代表存在 2代表删除)', + `create_by` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='部门表'; + +DROP TABLE IF EXISTS `dss_sidebar`; +CREATE TABLE `dss_sidebar` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `workspace_id` int(11) DEFAULT '0' COMMENT '空间ID,默认为0,所有空间都有', + `name` varchar(200) DEFAULT NULL COMMENT '名称', + `name_en` varchar(400) DEFAULT NULL COMMENT '名称(英文)', + `title` varchar(200) DEFAULT NULL COMMENT '标题', + `title_en` varchar(400) DEFAULT NULL COMMENT '标题(英文)', + `type` int(1) NOT NULL COMMENT '类型: 0-知识库,1-菜单,2-常见问题', + `order_num` int(2) DEFAULT '1' COMMENT '序号,按照这个字段顺序显示', + `remark` varchar(200) DEFAULT NULL COMMENT '备注', + `create_user` varchar(100) DEFAULT NULL COMMENT '创建人', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `idx_workspace_id` (`workspace_id`) +) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8 COMMENT='侧边栏表'; + +DROP TABLE IF EXISTS `dss_sidebar_content`; +CREATE TABLE `dss_sidebar_content` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `workspace_id` int(11) DEFAULT '0' COMMENT '空间ID,默认为0,所有空间都有', + `sidebar_id` int(11) NOT NULL COMMENT '侧边栏ID', + `name` varchar(200) DEFAULT NULL COMMENT '名称', + `name_en` varchar(400) DEFAULT NULL COMMENT '名称(英文)', + `title` varchar(200) DEFAULT NULL COMMENT '标题', + `title_en` varchar(400) DEFAULT NULL COMMENT '标题(英文)', + `url` varchar(200) DEFAULT NULL COMMENT 'url', + `url_type` int(1) DEFAULT '0' COMMENT 'url类型: 0-内部系统,1-外部系统;默认是内部', + `icon` varchar(200) DEFAULT NULL COMMENT '图标', + `order_num` int(2) DEFAULT '1' COMMENT '序号,按照这个字段顺序显示', + `remark` varchar(200) DEFAULT NULL COMMENT '备注', + `create_user` varchar(100) DEFAULT NULL COMMENT '创建人', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `idx_sidebarws_id` (`workspace_id`,`sidebar_id`) +) ENGINE=InnoDB AUTO_INCREMENT=10 DEFAULT CHARSET=utf8 COMMENT='侧边栏-内容表'; + +DROP TABLE IF EXISTS `dss_workspace_download_audit`; +CREATE TABLE `dss_workspace_download_audit` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `creator` varchar(255) COMMENT '创建者', + `tenant` varchar(255) COMMENT '租户', + `path` varchar(255) COMMENT '文件路径', + `sql` varchar(3000) COMMENT '执行sql脚本', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE = INNODB DEFAULT CHARSET = utf8 COMMENT = '文件下载审计'; + +/* +--------------------------------------------------------------------- +--------------------------- DSS Workflow --------------------------- +--------------------------------------------------------------------- +*/ + +DROP TABLE IF EXISTS `dss_workflow`; +CREATE TABLE `dss_workflow` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `state` tinyint(1) DEFAULT NULL, + `source` varchar(255) DEFAULT NULL, + `description` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `creator` varchar(32) DEFAULT NULL, + `is_root_flow` tinyint(1) DEFAULT NULL, + `rank` int(10) DEFAULT NULL, + `project_id` bigint(20) DEFAULT NULL, + `has_saved` tinyint(1) DEFAULT NULL, + `uses` varchar(255) DEFAULT NULL, + `bml_version` varchar(255) DEFAULT NULL, + `resource_id` varchar(255) DEFAULT NULL, + `linked_appconn_names` varchar(255) DEFAULT NULL, + `dss_labels` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=455 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + +DROP TABLE IF EXISTS `dss_workflow_relation`; +CREATE TABLE `dss_workflow_relation` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `flow_id` bigint(20) DEFAULT NULL, + `parent_flow_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=78 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + + +DROP TABLE IF EXISTS `dss_workflow_edit_lock`; +CREATE TABLE `dss_workflow_edit_lock` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '', + `flow_id` bigint(11) NOT NULL COMMENT '', + `username` varchar(64) NOT NULL COMMENT '', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `owner` varchar(128) NOT NULL COMMENT '', + `is_expire` tinyint(1) NOT NULL DEFAULT '0' COMMENT '', + `lock_content` varchar(512) NOT NULL COMMENT '', + PRIMARY KEY (`id`), + UNIQUE KEY `dss_workflow_edit_lock_flow_id_IDX` (`flow_id`) USING BTREE + ) ENGINE=InnoDB AUTO_INCREMENT=571 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workflow_task`; +CREATE TABLE `dss_workflow_task` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Primary Key, auto increment', + `instance` varchar(50) DEFAULT NULL COMMENT 'An instance of Entrance, consists of IP address of the entrance server and port', + `exec_id` varchar(50) DEFAULT NULL COMMENT 'execution ID, consists of jobID(generated by scheduler), executeApplicationName , creator and instance', + `um_user` varchar(50) DEFAULT NULL COMMENT 'User name', + `submit_user` varchar(50) DEFAULT NULL COMMENT 'submitUser name', + `execution_code` text COMMENT 'Run script. When exceeding 6000 lines, script would be stored in HDFS and its file path would be stored in database', + `progress` float DEFAULT NULL COMMENT 'Script execution progress, between zero and one', + `log_path` varchar(200) DEFAULT NULL COMMENT 'File path of the log files', + `result_location` varchar(200) DEFAULT NULL COMMENT 'File path of the result', + `status` varchar(50) DEFAULT NULL COMMENT 'Script execution status, must be one of the following: Inited, WaitForRetry, Scheduled, Running, Succeed, Failed, Cancelled, Timeout', + `created_time` datetime DEFAULT NULL COMMENT 'Creation time', + `updated_time` datetime DEFAULT NULL COMMENT 'Update time', + `run_type` varchar(50) DEFAULT NULL COMMENT 'Further refinement of execution_application_time, e.g, specifying whether to run pySpark or SparkR', + `err_code` int(11) DEFAULT NULL COMMENT 'Error code. Generated when the execution of the script fails', + `err_desc` text COMMENT 'Execution description. Generated when the execution of script fails', + `execute_application_name` varchar(200) DEFAULT NULL COMMENT 'The service a user selects, e.g, Spark, Python, R, etc', + `request_application_name` varchar(200) DEFAULT NULL COMMENT 'Parameter name for creator', + `script_path` varchar(200) DEFAULT NULL COMMENT 'Path of the script in workspace', + `params` text COMMENT 'Configuration item of the parameters', + `engine_instance` varchar(50) DEFAULT NULL COMMENT 'An instance of engine, consists of IP address of the engine server and port', + `task_resource` varchar(1024) DEFAULT NULL, + `engine_start_time` time DEFAULT NULL, + `label_json` varchar(200) DEFAULT NULL COMMENT 'label json', + PRIMARY KEY (`id`), + KEY `created_time` (`created_time`), + KEY `um_user` (`um_user`) +) ENGINE=InnoDB AUTO_INCREMENT=715 DEFAULT CHARSET=utf8mb4; + +DROP TABLE IF EXISTS `dss_workflow_execute_info`; +CREATE TABLE `dss_workflow_execute_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_id` bigint(20) NOT NULL COMMENT '任务id', + `status` int(1) DEFAULT NULL COMMENT '状态,0:失败 1:成功,', + `flow_id` bigint(20) NOT NULL COMMENT 'flowId', + `version` varchar(200) DEFAULT NULL COMMENT '工作流bml版本号', + `failed_jobs` text COMMENT '执行失败节点', + `Pending_jobs` text COMMENT '未执行节点', + `skipped_jobs` text COMMENT '执行跳过节点', + `succeed_jobs` text COMMENT '执行成功节点', + `createtime` datetime NOT NULL COMMENT '创建时间', + `running_jobs` text COMMENT '正在执行节点', + `updatetime` datetime DEFAULT NULL COMMENT '更新时间', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB AUTO_INCREMENT=471 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workflow_node`; +CREATE TABLE `dss_workflow_node` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(16) DEFAULT NULL, + `appconn_name` varchar(64) DEFAULT '-1' COMMENT 'appconn的名称,与dss_appconn这表的appconn_name名称对应', + `node_type` varchar(255) DEFAULT NULL, + `jump_type` int(11) DEFAULT NULL, + `support_jump` tinyint(1) DEFAULT NULL, + `submit_to_scheduler` tinyint(1) DEFAULT NULL, + `enable_copy` tinyint(1) DEFAULT NULL, + `should_creation_before_node` tinyint(1) DEFAULT NULL, + `icon_path` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB AUTO_INCREMENT=21 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workflow_node_group`; +CREATE TABLE `dss_workflow_node_group` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(32) NOT NULL, + `name_en` varchar(32) NOT NULL, + `description` varchar(255) DEFAULT NULL, + `order` tinyint(2) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workflow_node_to_group`; +CREATE TABLE `dss_workflow_node_to_group` ( + `node_id` int(11) NOT NULL, + `group_id` int(11) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workflow_node_to_ui`; +CREATE TABLE `dss_workflow_node_to_ui` ( + `workflow_node_id` int(11) NOT NULL, + `ui_id` int(11) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workflow_node_ui`; +CREATE TABLE `dss_workflow_node_ui` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `key` varchar(64) NOT NULL, + `description` varchar(255) DEFAULT NULL, + `description_en` varchar(255) DEFAULT NULL, + `lable_name` varchar(64) NOT NULL, + `lable_name_en` varchar(64) NOT NULL, + `ui_type` varchar(16) NOT NULL, + `required` tinyint(1) NOT NULL, + `value` varchar(255) DEFAULT NULL, + `default_value` varchar(255) DEFAULT NULL, + `is_hidden` tinyint(1) NOT NULL, + `condition` varchar(255) DEFAULT NULL, + `is_advanced` tinyint(1) NOT NULL, + `order` tinyint(2) NOT NULL, + `node_menu_type` tinyint(1) NOT NULL, + `is_base_info` tinyint(1) NOT NULL, + `position` varchar(32) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=45 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workflow_node_ui_to_validate`; +CREATE TABLE `dss_workflow_node_ui_to_validate` ( + `ui_id` int(11) NOT NULL, + `validate_id` int(11) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +DROP TABLE IF EXISTS `dss_workflow_node_ui_validate`; +CREATE TABLE `dss_workflow_node_ui_validate` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `validate_type` varchar(16) NOT NULL, + `validate_range` varchar(255) DEFAULT NULL, + `error_msg` varchar(255) DEFAULT NULL, + `error_msg_en` varchar(255) DEFAULT NULL, + `trigger` varchar(16) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=59 DEFAULT CHARSET=utf8mb4; + +DROP TABLE IF EXISTS `dss_workspace_user_favorites_appconn`; +CREATE TABLE `dss_workspace_user_favorites_appconn` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `username` varchar(64) DEFAULT NULL, + `workspace_id` bigint(20) DEFAULT '1', + `menu_appconn_id` int(20) DEFAULT NULL, + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + `type` varchar(20) NOT NULL DEFAULT "" COMMENT "dingyiding or 收藏", + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=94 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workspace_menu_role`; +CREATE TABLE `dss_workspace_menu_role` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `workspace_id` int(20) DEFAULT NULL, + `menu_id` int(20) DEFAULT NULL, + `role_id` int(20) DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `updateby` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=5263 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workspace_menu`; +CREATE TABLE `dss_workspace_menu` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `name` varchar(64) DEFAULT NULL, + `title_en` varchar(64) DEFAULT NULL, + `title_cn` varchar(64) DEFAULT NULL, + `description` varchar(255) DEFAULT NULL, + `is_active` tinyint(1) DEFAULT '1', + `icon` varchar(255) DEFAULT NULL, + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workspace_appconn_role`; +CREATE TABLE `dss_workspace_appconn_role` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `appconn_id` int(20) DEFAULT NULL, + `role_id` int(20) DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `updateby` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=5103 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_workspace_menu_appconn`; +CREATE TABLE `dss_workspace_menu_appconn` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `appconn_id` int(20) DEFAULT NULL, + `menu_id` int(20) NOT NULL, + `title_en` varchar(64) DEFAULT NULL, + `title_cn` varchar(64) DEFAULT NULL, + `desc_en` varchar(255) DEFAULT NULL, + `desc_cn` varchar(255) DEFAULT NULL, + `labels_en` varchar(255) DEFAULT NULL, + `labels_cn` varchar(255) DEFAULT NULL, + `is_active` tinyint(1) DEFAULT NULL, + `access_button_en` varchar(64) DEFAULT NULL, + `access_button_cn` varchar(64) DEFAULT NULL, + `manual_button_en` varchar(64) DEFAULT NULL, + `manual_button_cn` varchar(64) DEFAULT NULL, + `manual_button_url` varchar(255) DEFAULT NULL, + `icon` varchar(255) DEFAULT NULL, + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + `image` varchar(200) DEFAULT NULL COMMENT '图片', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_workspace_user_role`; -- use this table +CREATE TABLE `dss_workspace_user_role` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `username` varchar(32) DEFAULT NULL, + `role_id` int(20) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `created_by` varchar(255) DEFAULT NULL, + `user_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 comment '空间用户角色关系表'; + + +DROP TABLE IF EXISTS `dss_proxy_user`; +CREATE TABLE `dss_proxy_user` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `username` varchar(64) DEFAULT NULL, + `proxy_user_name` varchar(64) DEFAULT NULL, + `create_by` varchar(64) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `remark` varchar(500) DEFAULT NULL COMMENT '备注', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=214 DEFAULT CHARSET=utf8; + diff --git a/db/dss_dml.sql b/db/dss_dml.sql new file mode 100644 index 000000000..7ae770c63 --- /dev/null +++ b/db/dss_dml.sql @@ -0,0 +1,416 @@ +DELETE FROM dss_appconn; +INSERT INTO `dss_appconn` (`id`, `appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) +VALUES (1,'sso',0,1,0,0,NULL,"com.webank.wedatasphere.dss.appconn.sso.SSOAppConn",NULL,NULL), +(2,'scriptis',0,1,0,0,"sso",NULL,NULL,NULL), +(3,'workflow',0,1,1,0,NULL,'com.webank.wedatasphere.dss.appconn.workflow.WorkflowAppConn','/appcom/Install/dss/dss-appconns/workflow',NULL), +(4,'apiservice',0,1,0,0,"sso",NULL,NULL,NULL); + +DELETE FROM dss_appconn_instance; +select @scriptis_appconn_id:= id from dss_appconn where appconn_name="scriptis"; +select @workflow_appconn_id:= id from dss_appconn where appconn_name="workflow"; +select @apiservice_appconn_id:= id from dss_appconn where appconn_name="apiservice"; +INSERT INTO `dss_appconn_instance` (`id`, `appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`) +VALUES (2, @scriptis_appconn_id, 'DEV', '/home', '', ''), +(3, @workflow_appconn_id,'DEV','/workspaceHome','',''), +(4, @apiservice_appconn_id, 'DEV', '/apiservices', '', ''); + +DELETE FROM dss_workspace; +insert into `dss_workspace`(`id`, `name`,`label`,`description`,`create_by`,`create_time`,`department`,`product`,`source`,`last_update_time`,`last_update_user`,`workspace_type`) +values(224, 'bdapWorkspace','','bdapWorkspace','hadoop','2020-07-13 02:39:41','1','bdapWorkspace',NULL,'2020-07-13 02:39:41','hadoop','project'); + +DELETE FROM dss_user; +INSERT INTO `dss_user` VALUES (100,'hadoop_test','hadoop_test',1,101,1,'','','','0',NULL,'2021-11-17 09:33:45','2021-11-17 09:51:55',NULL), +(215,'hadoop','hadoop',1,101,1,'','','','0',NULL,'2021-11-17 09:43:41','2021-11-17 09:51:49',NULL); + +DELETE FROM dss_workspace_dictionary; +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (1,0,'0','空间开发流程','Space development process','w_develop_process',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'空间开发流程','SYSTEM','2020-12-28 17:32:34',NULL,'2021-02-22 17:46:40'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (2,0,'w_develop_process','需求','Demand','wdp_demand','创建新的业务需求,并将需求指派给对应负责人。','Create new business requirements and assign them to the corresponding responsible person.','Demo案例','Demo case',NULL,0,'xuqiu',1,'空间开发流程-需求','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-23 09:38:07'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (3,0,'w_develop_process','设计','Design','wdp_design','针对新的业务需求,进行数仓规划和库表设计。','According to the new business requirements, data warehouse planning and database table design are carried out.','Demo案例','Demo case',NULL,0,'sheji',1,'空间开发流程-设计','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-23 09:38:09'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (4,0,'w_develop_process','开发','Development','wdp_development','针对新的业务需求,进行数仓规划和库表设计。','According to the new business requirements, data warehouse planning and database table design are carried out.','Demo案例','Demo case',NULL,0,'kaifa',1,'空间开发流程-开发','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-23 09:38:10'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (5,0,'w_develop_process','调试','Debugging','wdp_debug','创建新的业务需求,并将需求指派给对应负责人。','Create new business requirements and assign them to the corresponding responsible person.','Demo案例','Demo case',NULL,0,'tiaoshi',1,'空间开发流程-调试','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-23 09:38:11'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (6,0,'w_develop_process','生产','Production','wdp_product','创建新的业务需求,并将需求指派给对应负责人。','Create new business requirements and assign them to the corresponding responsible person.','Demo案例','Demo case',NULL,0,'shengchan',1,'空间开发流程-生产','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-23 09:38:12'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (7,0,'0','工程开发流程','Engineering development process','p_develop_process',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'工程开发流程','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:48:48'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (8,0,'p_develop_process','开发中心','Development Center','pdp_development_center','dev',NULL,NULL,NULL,NULL,0,'kaifa-icon',1,'工程开发流程-开发中心','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:49:02'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (10,0,'0','工程编排模式','Project layout mode','p_orchestrator_mode',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'工程编排模式','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:49:36'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (11,0,'p_orchestrator_mode','工作流','Workflow','pom_work_flow','radio',NULL,NULL,NULL,NULL,0,'gongzuoliu-icon',1,'工程编排模式-工作流','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:49:49'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (14,0,'pom_work_flow','DAG','DAG','pom_work_flow_DAG',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'工程编排模式-工作流-DAG','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:50:31'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (16,0,'pom_single_task','Scriptis','Scriptis','pom_single_task_scriptis',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'工程编排模式-单任务-Scriptis','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:51:08'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (18,0,'pom_single_task','Qualitis','Qualitis','pom_single_task_qualitis',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'工程编排模式-单任务-Qualitis','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:50:53'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (20,0,'pom_consist_orchestrator','Qualitis','Qualitis','pom_consist_orchestrator_qualitis',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'工程编排模式-组合编排-Qualitis','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:50:57'); +insert into `dss_workspace_dictionary`(`id`,`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (21,0,'pom_consist_orchestrator','Email','Email','pom_consist_orchestrator_email',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'工程编排模式-组合编排-Email','SYSTEM','2020-12-28 17:32:35',NULL,'2021-02-22 17:51:22'); +insert into `dss_workspace_dictionary`(`workspace_id`, `parent_key`, `dic_name`, `dic_name_en`, `dic_key`, `dic_value`, `dic_value_en`, `title`, `title_en`, `url`, `url_type`, `icon`, `order_num`, `remark`, `create_user`, `create_time`, `update_user`, `update_time`) values('0','0','工作空间默认部门','Space development name','w_workspace_department','10001-部门一;10002-部门二;10003-部门三',NULL,NULL,NULL,NULL,'0',NULL,'1','工作空间默认部门,前面是id后面是部门名称中间使用‘-’,横杆分隔,多个以英文分号分隔','SYSTEM','2020-12-28 17:32:34',NULL,'2021-02-22 17:46:40'); + +DELETE FROM dss_sidebar; +insert into `dss_sidebar`(`id`,`workspace_id`,`name`,`name_en`,`title`,`title_en`,`type`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (2,0,'菜单','Menu','菜单','Menu',1,1,NULL,'SYSTEM','2020-12-15 13:21:06',NULL,'2021-02-23 09:45:50'); +-- insert into `dss_sidebar`(`id`,`workspace_id`,`name`,`name_en`,`title`,`title_en`,`type`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (3,0,'常见问题','Common problem','常见问题','Common problem',1,1,NULL,'SYSTEM','2020-12-15 13:21:06',NULL,'2021-02-23 09:46:18'); + +DELETE FROM dss_sidebar_content; +insert into `dss_sidebar_content`(`id`,`workspace_id`,`sidebar_id`,`name`,`name_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (3,0,2,NULL,NULL,'工作空间管理','Workspace management','/workspaceManagement/productsettings',0,'menuIcon',1,NULL,'SYSTEM','2020-12-15 13:21:07',NULL,'2021-02-23 09:47:49'); +insert into `dss_sidebar_content`(`id`,`workspace_id`,`sidebar_id`,`name`,`name_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (4,0,2,NULL,NULL,'UDF管理','UDF management','dss/linkis/?noHeader=1&noFooter=1#/urm/udfManagement',1,'menuIcon',1,NULL,'SYSTEM','2020-12-15 13:21:07',NULL,'2021-02-23 09:47:11'); +-- insert into `dss_sidebar_content`(`id`,`workspace_id`,`sidebar_id`,`name`,`name_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (5,0,3,NULL,NULL,'资源配置说明',NULL,'http://127.0.0.1:8088/kn/d/38',1,'fi-warn',1,NULL,'SYSTEM','2020-12-15 13:21:07',NULL,'2021-01-12 17:16:52'); +-- insert into `dss_sidebar_content`(`id`,`workspace_id`,`sidebar_id`,`name`,`name_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (6,0,3,NULL,NULL,'Spark使用指南','[Discussion on error code 22223]','http://127.0.0.1:8088/kn/d/40',1,'fi-warn',1,NULL,'SYSTEM','2020-12-15 13:21:07',NULL,'2021-02-23 09:48:28'); +-- insert into `dss_sidebar_content`(`id`,`workspace_id`,`sidebar_id`,`name`,`name_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (7,0,3,NULL,NULL,'Hive语法介绍',NULL,'http://127.0.0.1:8088/kn/d/34',1,'fi-warn',1,NULL,'SYSTEM','2020-12-15 13:21:07',NULL,'2021-01-12 17:17:00'); +-- insert into `dss_sidebar_content`(`id`,`workspace_id`,`sidebar_id`,`name`,`name_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (8,0,3,NULL,NULL,'工作流使用介绍',NULL,'http://127.0.0.1:8088/kn/d/42',1,'fi-warn',1,NULL,'SYSTEM','2020-12-15 13:21:07',NULL,'2021-01-12 17:17:01'); +-- insert into `dss_sidebar_content`(`id`,`workspace_id`,`sidebar_id`,`name`,`name_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (9,0,3,NULL,NULL,'数据服务使用介绍','Discussion on error code 22223','http://127.0.0.1:8088/kn/d/32',1,'fi-warn',1,NULL,'SYSTEM','2020-12-15 13:21:07',NULL,'2021-02-23 09:48:19'); + +DELETE FROM dss_workspace_menu; +INSERT INTO `dss_workspace_menu` (`id`, `name`, `title_en`, `title_cn`, `description`, `is_active`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`) VALUES('1','数据交换','data exchange','数据交换','数据交换描述','1',NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO `dss_workspace_menu` (`id`, `name`, `title_en`, `title_cn`, `description`, `is_active`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`) VALUES('2','数据分析','data analysis','数据分析','数据分析描述','1',NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO `dss_workspace_menu` (`id`, `name`, `title_en`, `title_cn`, `description`, `is_active`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`) VALUES('3','生产运维','production operation','生产运维','生产运维描述','1',NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO `dss_workspace_menu` (`id`, `name`, `title_en`, `title_cn`, `description`, `is_active`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`) VALUES('4','数据质量','data quality','数据质量','数据质量描述','1',NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO `dss_workspace_menu` (`id`, `name`, `title_en`, `title_cn`, `description`, `is_active`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`) VALUES('5','管理员功能','administrator function','管理员功能','管理员功能描述','0',NULL,NULL,NULL,NULL,NULL,NULL); +insert into `dss_workspace_menu` (`id`, `name`, `title_en`, `title_cn`, `description`, `is_active`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`) values('6','数据应用','data application','数据应用','数据应用描述','1',NULL,NULL,NULL,NULL,NULL,NULL); +insert into `dss_workspace_menu` (`id`, `name`, `title_en`, `title_cn`, `description`, `is_active`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`) values('7','应用开发','application development','应用开发','应用开发描述','1',NULL,NULL,NULL,NULL,NULL,NULL); + +DELETE FROM dss_workspace_menu_appconn; +INSERT INTO dss_workspace_menu_appconn (appconn_id, menu_id, title_en, title_cn, desc_en, desc_cn, labels_en, labels_cn, is_active, access_button_en, access_button_cn, manual_button_en, manual_button_cn, manual_button_url, icon, `order`, create_by, create_time, last_update_time, last_update_user, image) +VALUES (@scriptis_appconn_id, 2, 'Scriptis', 'Scriptis', 'Scriptis is a one-stop interactive data exploration analysis tool built by WeDataSphere, uses Linkis as the kernel.', 'Scriptis是微众银行微数域(WeDataSphere)打造的一站式交互式数据探索分析工具,以任意桥(Linkis)做为内核,提供多种计算存储引擎(如Spark、Hive、TiSpark等)、Hive数据库管理功能、资源(如Yarn资源、服务器资源)管理、应用管理和各种用户资源(如UDF、变量等)管理的能力。', 'scripts development,IDE', '脚本开发,IDE', 1, 'enter Scriptis', '进入Scriptis', 'user manual', '用户手册', 'http://127.0.0.1:8088/wiki/scriptis/manual/workspace_cn.html', 'shujukaifa-logo', null, null, null, null, null, 'shujukaifa-icon'); +INSERT INTO dss_workspace_menu_appconn (appconn_id, menu_id, title_en, title_cn, desc_en, desc_cn, labels_en, labels_cn, is_active, access_button_en, access_button_cn, manual_button_en, manual_button_cn, manual_button_url, icon, `order`, create_by, create_time, last_update_time, last_update_user, image) +VALUES (@workflow_appconn_id, 2, 'workflow', '工作流开发', '工作流开发', '工作流开发', null, null, 1, 'Enter workflow', '进入 工作流开发', null, null, null, null, null, null, null, null, null, null); +INSERT INTO dss_workspace_menu_appconn (appconn_id, menu_id, title_en, title_cn, desc_en, desc_cn, labels_en, labels_cn, is_active, access_button_en, access_button_cn, manual_button_en, manual_button_cn, manual_button_url, icon, `order`, create_by, create_time, last_update_time, last_update_user, image) +VALUES (@apiservice_appconn_id, 7, 'dataService', '数据服务', '/dataService', '/dataService', null, null, 1, 'Enter dataService', '进入 数据服务', null, null, null, null, null, null, null, null, null, null); + + +DELETE FROM dss_workspace_role; +INSERT INTO `dss_workspace_role` (`id`, `workspace_id`, `name`, `front_name`, `update_time`, `description`) VALUES('1','-1','admin','管理员','2020-07-13 02:43:35','通用角色管理员'); +INSERT INTO `dss_workspace_role` (`id`, `workspace_id`, `name`, `front_name`, `update_time`, `description`) VALUES('2','-1','maintenance','运维用户','2020-07-13 02:43:35','通用角色运维用户'); +INSERT INTO `dss_workspace_role` (`id`, `workspace_id`, `name`, `front_name`, `update_time`, `description`) VALUES('3','-1','developer','开发用户','2020-07-13 02:43:35','通用角色开发用户'); +INSERT INTO `dss_workspace_role` (`id`, `workspace_id`, `name`, `front_name`, `update_time`, `description`) VALUES('4','-1','analyser','分析用户','2020-07-13 02:43:36','通用角色分析用户'); +INSERT INTO `dss_workspace_role` (`id`, `workspace_id`, `name`, `front_name`, `update_time`, `description`) VALUES('5','-1','operator','运营用户','2020-07-13 02:43:36','通用角色运营用户'); +INSERT INTO `dss_workspace_role` (`id`, `workspace_id`, `name`, `front_name`, `update_time`, `description`) VALUES('6','-1','boss','领导','2020-07-13 02:43:36','通用角色领导'); +INSERT INTO `dss_workspace_role` (`id`, `workspace_id`, `name`, `front_name`, `update_time`, `description`) VALUES('7','-1','apiUser','数据服务用户','2020-08-21 11:35:02','通用角色数据服务用户'); + +DELETE FROM dss_workspace_user_role; +select @defaultWorkspaceId:=id from dss_workspace where name='bdapWorkspace'; +insert into `dss_workspace_user_role`(`workspace_id`,`username`,`role_id`,`create_time`,`created_by`,`user_id`) values +(@defaultWorkspaceId,'hadoop',1,'2021-09-06 14:39:17','hadoop',0),(@defaultWorkspaceId,'hadoop',2,'2021-09-06 14:39:17','hadoop',0), +(@defaultWorkspaceId,'hadoop',3,'2021-09-06 14:39:17','hadoop',0),(@defaultWorkspaceId,'hadoop',4,'2021-09-06 14:39:17','hadoop',0), +(@defaultWorkspaceId,'hadoop',5,'2021-09-06 14:39:17','hadoop',0),(@defaultWorkspaceId,'hadoop',6,'2021-09-06 14:39:17','hadoop',0), +(@defaultWorkspaceId,'hadoop',7,'2021-09-06 14:39:17','hadoop',0); + +DELETE FROM dss_workflow_node; +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('1','python','scriptis','linkis.python.python','2','1','1','1','0','icons/python.icon'); +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('2','pyspark','scriptis','linkis.spark.py','2','1','1','1','0','icons/pyspark.icon'); +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('3','sql','scriptis','linkis.spark.sql','2','1','1','1','0','icons/sql.icon'); +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('4','scala','scriptis','linkis.spark.scala','2','1','1','1','0','icons/scala.icon'); +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('5','hql','scriptis','linkis.hive.hql','2','1','1','1','0','icons/hql.icon'); +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('6','jdbc','scriptis','linkis.jdbc.jdbc','2','1','1','1','0','icons/jdbc.icon'); +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('7','shell','scriptis','linkis.shell.sh','2','1','1','1','0','icons/shell.icon'); +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('10','connector','scriptis','linkis.control.empty','2','0','1','1','0','icons/connector.icon'); +insert into `dss_workflow_node` (`id`, `name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('12','subFlow','scriptis','workflow.subflow','2','1','1','0','1','icons/subFlow.icon'); + +DELETE FROM dss_workflow_node_group; +insert into `dss_workflow_node_group`(`id`,`name`,`name_en`,`description`,`order`) values (1,'数据交换','Data exchange',NULL,1); +insert into `dss_workflow_node_group`(`id`,`name`,`name_en`,`description`,`order`) values (2,'数据开发','Data development',NULL,2); +insert into `dss_workflow_node_group`(`id`,`name`,`name_en`,`description`,`order`) values (3,'数据质量','Data Givernance',NULL,3); +insert into `dss_workflow_node_group`(`id`,`name`,`name_en`,`description`,`order`) values (4,'数据可视化','Data visualization',NULL,4); +insert into `dss_workflow_node_group`(`id`,`name`,`name_en`,`description`,`order`) values (5,'数据输出','Data output',NULL,8); +insert into `dss_workflow_node_group`(`id`,`name`,`name_en`,`description`,`order`) values (6,'信号节点','Signal node',NULL,6); +insert into `dss_workflow_node_group`(`id`,`name`,`name_en`,`description`,`order`) values (7,'功能节点','Function node',NULL,7); +insert into `dss_workflow_node_group`(`id`,`name`,`name_en`,`description`,`order`) values (8,'机器学习','Machine Learning',NULL,5); + +DELETE FROM dss_workflow_node_to_group; +select @scriptis_node_groupId:=id from dss_workflow_node_group where name='数据开发'; +select @function_node_groupId:=id from dss_workflow_node_group where name='功能节点'; +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (1, @scriptis_node_groupId); +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (2, @scriptis_node_groupId); +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (3, @scriptis_node_groupId); +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (4, @scriptis_node_groupId); +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (5, @scriptis_node_groupId); +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (6, @scriptis_node_groupId); +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (7, @scriptis_node_groupId); +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (10, @function_node_groupId); +insert into `dss_workflow_node_to_group`(`node_id`,`group_id`) values (12, @function_node_groupId); + +DELETE FROM dss_workflow_node_ui; +-- todo msg.topic在receiver和sender使用了重复key +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (1,'title','请填写节点名称','Please enter node name','节点名','Node name','Input',1,NULL,NULL,0,NULL,0,1,1,1,'node'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (3,'desc','请填写节点描述','Please enter the node description','节点描述','Node description','Text',0,NULL,NULL,0,NULL,0,4,1,1,'node'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (5,'businessTag',NULL,NULL,'业务标签','businessTag','Tag',0,NULL,NULL,0,NULL,0,2,1,1,'node'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (6,'appTag',NULL,NULL,'应用标签','appTag','Tag',0,NULL,NULL,0,NULL,0,3,1,1,'node'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (7,'spark.driver.memory','驱动器内存大小,默认值:2','Driver memory, default value: 2','spark-driver-memory','spark-driver-memory','Input',0,NULL,'2',0,NULL,0,1,1,0,'startup'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (8,'spark.executor.memory','执行器内存大小,默认值:3','Executor memory, default value: 3','spark-executor-memory','spark-executor-memory','Input',0,NULL,'3',0,NULL,0,1,1,0,'startup'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (9,'spark.executor.cores','执行器核心个数,默认值:1','Number of cores per executor, default value: 1','spark-executor-cores','spark-executor-cores','Input',0,NULL,'2',0,NULL,0,1,1,0,'startup'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (10,'spark.executor.instances','执行器个数,默认值:2','Number of executors, default value: 2','spark-executor-instances','spark-executor-instances','Input',0,NULL,'2',0,NULL,0,1,1,0,'startup'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (11,'wds.linkis.rm.yarnqueue','执行队列','Execution queue','wds-linkis-yarnqueue','wds-linkis-yarnqueue','Input',0,NULL,'dws',0,NULL,0,1,1,0,'startup'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (12,'resources',NULL,NULL,'资源信息','Resource information','Upload',0,'[]',NULL,0,NULL,0,1,1,0,'node'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (13,'category','请选择类型','Please choose the type','类型','Type','Select',1,'[\"node\"]','node',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (14,'subject','请填写邮件标题','Please enter the email subject','邮件标题','Email Subject','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (15,'content','请选择或输入发送项','Please choose or enter the items to send','发送项','Intems to Send','MultiBinding',1,'[\"linkis.appconn.visualis.display\",\"linkis.appconn.visualis.dashboard\"]','[]',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (16,'to','请填写收件人','Please enter recipients','收件人','To','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (17,'cc','请填写抄送人','Please enter carbon copy recipients','抄送','Cc','Input',0,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (18,'bcc','请填写秘密发送人','Please enter blind carbon copy recipients','秘密抄送','Bcc','Input',0,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (19,'itsm','请填写关联审批单','Please enter ITSM','关联审批单','ITSM','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (20,'msg.type','请正确填写消息类型','Please enter message type correctly','msg.type','msg.type','Disable',1,NULL,'SEND',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (21,'msg.topic','消息主题,必须与eventreceiver完全一致','Message subject must be exactly the same as eventreceiver','msg.topic','msg.topic','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (22,'msg.sender','请正确填写发送者','Please enter the sender correctly','msg.sender','msg.sender','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (23,'msg.name','消息名称,必须与eventreceiver完全一致','The message name must be exactly the same as the eventreceiver','msg.name','msg.name','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (24,'msg.body','请正确填写消息内容','Please enter the message content correctly','msg.body','msg.body','Text',0,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (25,'msg.type','请正确填写消息类型','Please enter message type correctly','msg.type','msg.type','Disable',1,NULL,'RECEIVE',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (26,'msg.receiver','请正确填写消息接收者','Please enter message recipients correctly','msg.receiver','msg.receiver','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (27,'query.frequency','请填写查询频率,默认10次','Please enter query frequency, 10 times by default','query.frequency','query.frequency','Disable',0,NULL,'10',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (28,'max.receive.hours','请填写等待时间,默认1小时','Please enter waiting time, 1 hour by default','max.receive.hours','max.receive.hours','Input',0,NULL,'12',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (29,'msg.savekey','消息共享key值,默认msg.body','The ky of message content, msg.body by default','msg.savekey','msg.savekey','Input',0,NULL,'msg.body',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (30,'only.receive.today',NULL,NULL,'only.receive.today','only.receive.today','Input',0,NULL,'true',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (31,'source.type','请选择数据来源','Please choose the data source','source.type','source.type','Select',1,'[\"hivedb\",\"maskdb\"]',NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (32,'check.object','比如:db.tb{ds=${run_date}}','Please enter the name of data dependency','check.object','check.object','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (33,'max.check.hours',NULL,NULL,'max.check.hours','max.check.hours','Input',0,NULL,'1',0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (34,'job.desc','请正确填写多源配置','Please enter multi-source configuration correctly','job.desc','job.desc','Text',0,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (35,'filter',NULL,NULL,'过滤条件','Filter','Input',0,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (37,'upStreams','请选择上游节点','Please select upstream node','绑定上游节点','Bind front node','Binding',1,'[\"*\"]','empty',0,NULL,0,3,0,1,'node'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (39,'msg.topic','消息主题,必须与eventsender完全一致','Message subject must be exactly the same as eventsender','msg.topic','msg.topic','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (40,'msg.name','消息名称,必须与eventsender完全一致','The message name must be exactly the same as the eventsender ','msg.name','msg.name','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (41,'executeUser','请填写执行用户','Please enter execute user','执行用户','executeUser','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +insert into `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (42,'Filter','请填写过滤条件','Please enter filter','过滤条件','Filter','Input',1,NULL,NULL,0,NULL,0,1,1,0,'runtime'); +INSERT INTO `dss_workflow_node_ui`(`id`,`key`,`description`,`description_en`,`lable_name`,`lable_name_en`,`ui_type`,`required`,`value`,`default_value`,`is_hidden`,`condition`,`is_advanced`,`order`,`node_menu_type`,`is_base_info`,`position`) values (45,'ReuseEngine','请选择是否复用引擎','Please choose to reuse engin or not','是否复用引擎','reuse-engine-or-not','Select',1,'[\"true\",\"false\"]','true',0,NULL,0,1,1,0,'startup'); + +DELETE FROM dss_workflow_node_to_ui; +select @workflow_node_sql:=id from dss_workflow_node where name='sql'; +select @workflow_node_python:=id from dss_workflow_node where name='python'; +select @workflow_node_pyspark:=id from dss_workflow_node where name='pyspark'; +select @workflow_node_scala:=id from dss_workflow_node where name='scala'; +select @workflow_node_hql:=id from dss_workflow_node where name='hql'; +select @workflow_node_shell:=id from dss_workflow_node where name='shell'; +select @workflow_node_jdbc:=id from dss_workflow_node where name='jdbc'; +select @workflow_node_connector:=id from dss_workflow_node where name='connector'; +select @workflow_node_subFlow:=id from dss_workflow_node where name='subFlow'; + +select @node_ui_title:=id from dss_workflow_node_ui where `key`='title' limit 1; +select @node_ui_desc:=id from dss_workflow_node_ui where `key`='desc' limit 1; +select @node_ui_businessTag:=id from dss_workflow_node_ui where `key`='businessTag'; +select @node_ui_appTag:=id from dss_workflow_node_ui where `key`='appTag'; +select @node_ui_spark_driver_memory:=id from dss_workflow_node_ui where `key`='spark.driver.memory'; +select @node_ui_spark_executor_memory:=id from dss_workflow_node_ui where `key`='spark.executor.memory'; +select @node_ui_spark_executor_cores:=id from dss_workflow_node_ui where `key`='spark.executor.cores'; +select @node_ui_spark_executor_instances:=id from dss_workflow_node_ui where `key`='spark.executor.instances'; +select @node_ui_wds_linkis_rm_yarnqueue:=id from dss_workflow_node_ui where `key`='wds.linkis.rm.yarnqueue'; +select @node_ui_resources:=id from dss_workflow_node_ui where `key`='resources'; +select @node_ui_category:=id from dss_workflow_node_ui where `key`='category'; +select @node_ui_subject:=id from dss_workflow_node_ui where `key`='subject'; +select @node_ui_content:=id from dss_workflow_node_ui where `key`='content'; +select @node_ui_to:=id from dss_workflow_node_ui where `key`='to'; +select @node_ui_cc:=id from dss_workflow_node_ui where `key`='cc'; +select @node_ui_bcc:=id from dss_workflow_node_ui where `key`='bcc'; +select @node_ui_itsm:=id from dss_workflow_node_ui where `key`='itsm'; +select @node_ui_msg_sender:=id from dss_workflow_node_ui where `key`='msg.sender'; +select @node_ui_msg_body:=id from dss_workflow_node_ui where `key`='msg.body'; +select @node_ui_msg_receiver:=id from dss_workflow_node_ui where `key`='msg.receiver'; +select @node_ui_query_frequency:=id from dss_workflow_node_ui where `key`='query.frequency'; +select @node_ui_max_receive_hours:=id from dss_workflow_node_ui where `key`='max.receive.hours'; +select @node_ui_msg_savekey:=id from dss_workflow_node_ui where `key`='msg.savekey'; +select @node_ui_only_receive_today:=id from dss_workflow_node_ui where `key`='only.receive.today'; +select @node_ui_source_type:=id from dss_workflow_node_ui where `key`='source.type'; +select @node_ui_check_object:=id from dss_workflow_node_ui where `key`='check.object'; +select @node_ui_max_check_hours:=id from dss_workflow_node_ui where `key`='max.check.hours'; +select @node_ui_job_desc:=id from dss_workflow_node_ui where `key`='job.desc'; +select @node_ui_upStreams:=id from dss_workflow_node_ui where `key`='upStreams'; +select @node_ui_executeUser:=id from dss_workflow_node_ui where `key`='executeUser'; +select @node_ui_ReuseEngine:=id from dss_workflow_node_ui where `key`='ReuseEngine'; + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_appTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_spark_driver_memory); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_spark_executor_memory); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_spark_executor_cores); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_spark_executor_instances); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_wds_linkis_rm_yarnqueue); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_sql,@node_ui_resources); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_sql,@node_ui_ReuseEngine); + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_python,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_python,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_python,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_python,@node_ui_appTag); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_python,@node_ui_ReuseEngine); + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_appTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_spark_driver_memory); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_spark_executor_memory); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_spark_executor_cores); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_spark_executor_instances); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_wds_linkis_rm_yarnqueue); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_pyspark,@node_ui_resources); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_pyspark,@node_ui_ReuseEngine); + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_appTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_spark_driver_memory); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_spark_executor_memory); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_spark_executor_cores); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_spark_executor_instances); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_scala,@node_ui_wds_linkis_rm_yarnqueue); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_scala,@node_ui_ReuseEngine); + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_hql,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_hql,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_hql,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_hql,@node_ui_appTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_hql,@node_ui_wds_linkis_rm_yarnqueue); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_hql,@node_ui_ReuseEngine); + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_shell,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_shell,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_shell,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_shell,@node_ui_appTag); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_shell,@node_ui_ReuseEngine); + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_jdbc,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_jdbc,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_jdbc,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_jdbc,@node_ui_appTag); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_jdbc,@node_ui_ReuseEngine); + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_connector,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_connector,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_connector,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_connector,@node_ui_appTag); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_connector,@node_ui_ReuseEngine); + +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_subFlow,@node_ui_title); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_subFlow,@node_ui_desc); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_subFlow,@node_ui_businessTag); +insert into `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@workflow_node_subFlow,@node_ui_appTag); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) VALUES (@workflow_node_subFlow,@node_ui_ReuseEngine); + +DELETE FROM dss_workflow_node_ui_validate; +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('7','NumInterval','[1,15]','驱动器内存大小,默认值:2','Drive memory size, default value: 2','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('8','NumInterval','[3,15]','执行器内存大小,默认值:3','Actuator memory size, default value: 3','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('9','NumInterval','[1,10]','执行器核心个数,默认值:1','Number of cores per executor, default value : 1','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('10','NumInterval','[1,40]','执行器个数,默认值:2','Number of per executor, default value : 2','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('13','OFT','[\"node\"]','请选择类型','Please select type','change'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('25','OFT','[\"RECEIVE\"]','','Please select ','change'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('27','NumInterval','[1,1000]','请填写查询频率,默认10次,范围:1-1000','Please fill in the inquiry frequency, default : 10, range is 1 to 1000','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('28','NumInterval','[1,1000]','请填写等待时间,默认1小时,范围:1-1000','Please enter waiting time, 1 hour by default, range is 1 to 1000','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('31','OFT','[\"hivedb\",\"maskdb\"]','','Invalid format,example:ProjectName@WFName@jobName','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('32','Regex','^[^\\u4e00-\\u9fa5]+$','此值不能输入中文','Chinese characters are not allowed','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('40','Regex','^[a-zA-Z]([^.]*\\.[^.]*){1,}$','需要检查的数据源dbname.tablename{partition}','Checked data source dbname.tablename{partition}','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('41','Regex','^[\\S\\n\\s]{0,500}$','长度在1到500个字符','The length is between 1 and 500 characters','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('44','Regex','^[a-zA-Z][a-zA-Z0-9_@-]*$','必须以字母开头,且只支持字母、数字、下划线、@、中横线','Started with alphabetic characters, only alphanumeric characters, underscore(_), @ and hyphen(-) are allowed','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('45','Regex','^[a-zA-Z0_9-]([^@]*@[^@]*){2}[a-zA-Z\\d]$','此值格式错误,例如:ProjectName@WFName@jobName','Invalid format,example:ProjectName@WFName@jobName','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('46','Regex','^[a-zA-Z0_9-]([^_]*_[^_]*){2}[a-zA-Z\\d]$','此值格式错误,例如:bdp_tac_name','Invalid format,example:bdp_tac_name','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('47','Regex','^.{1,128}$','长度在1到128个字符','The length is between 1 and 128 characters','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('48','Regex','^[a-zA-Z][a-zA-Z0-9_-]*$','必须以字母开头,且只支持字母、数字、下划线!','Started with alphabetic characters, only alphanumeric and underscore are allowed!','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('49','Regex','^[a-zA-Z0-9_\\u4e00-\\u9fa5]*$','只支持中文、字母、数字和下划线!','Only Chinese characters, alphanumeric characters and underscore are allowed in subject!','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('50','Regex','^[a-z][a-zA-Z0-9_.@;]*$','必须以字母开头,且只支持字母、数字、下划线、@、点','Must start with a letter and only letters, numbers, underscores, @, points are supported','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('51','Regex','^[0-9_.]*$','只支持数字、下划线、点','Only numbers, underscores and dots are supported','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('52','None',NULL,NULL,NULL,'blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('53','OFT','[\"SEND\"]',NULL,NULL,'change'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('54','NumInterval','[1,1000]','请填写等待时间,默认1小时,范围:1-1000','Please fill in the waiting time, default 1 hour, range: 1-1000','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('55','Required',NULL,'该值不能为空','The value cannot be empty\n\n','change'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('56','Function','validatorTitle','节点名不能和工作流名一样','The node name cannot be the same as the workflow name',NULL); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('57','Regex','^[a-zA-Z][a-zA-Z0-9_.-]*$','必须以字母开头,且只支持字母、数字、下划线、点!','It must start with a letter and only supports letters, numbers, underscores and dots!','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('58','Regex','(.+)@(.+)@(.+)','此格式错误,例如:ProjectName@WFName@jobName','Invalid format,example:ProjectName@WFName@jobName','blur'); +INSERT INTO `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('59','OFT','["true","false"]','请填写是否复用引擎,false:不复用,true:复用','Please fill in whether or not to reuse engine, true: reuse, false: not reuse','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('60', 'Regex', '^[0-9.]*g{0,1}$', 'Spark内存设置如2g', 'Drive memory size, default value: 2', 'blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('61','Regex','^(.|\s){1,500}$','长度在1到5000个字符','The length is between 1 and 5000 characters','blur'); +insert into `dss_workflow_node_ui_validate` (`id`, `validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) values('62','Regex','^.{1,150}$','长度在1到150个字符','The length is between 1 and 150 characters','blur'); + +DELETE FROM dss_workflow_node_ui_to_validate; +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_source_type,31); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_source_type,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_check_object,32); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_check_object,40); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_check_object,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_check_object,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_max_receive_hours,28); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_query_frequency,27); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_desc,32); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_desc,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_sender,44); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_sender,45); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_sender,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (25,25); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (25,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_receiver,32); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_receiver,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_receiver,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_receiver,58); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (21,32); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (21,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (21,46); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (21,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_savekey,32); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_savekey,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (23,47); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (23,48); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (23,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_body,32); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_body,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_spark_driver_memory,7); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_spark_executor_memory,8); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_spark_executor_cores,9); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_wds_linkis_rm_yarnqueue,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_wds_linkis_rm_yarnqueue,57); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_title,48); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_title,62); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_title,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_title,56); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_category,13); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_subject,47); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_to,50); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_cc,50); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_bcc,50); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_itsm,51); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_itsm,47); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_businessTag,52); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_appTag,52); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_resources,52); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (20,53); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (20,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_only_receive_today,52); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_max_check_hours,54); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_job_desc,32); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (35,52); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_sender,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_category,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_to,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_itsm,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_subject,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_upStreams,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (39,32); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (39,41); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (39,46); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (39,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (40,47); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (40,48); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (40,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_content,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_msg_sender,58); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_executeUser,55); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (42,52); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_ReuseEngine,59); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_spark_driver_memory,60); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_spark_executor_memory,60); +insert into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_job_desc,61); + + +DELETE FROM dss_workspace_appconn_role; +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@scriptis_appconn_id,'1','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@scriptis_appconn_id,'2','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@scriptis_appconn_id,'3','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@scriptis_appconn_id,'4','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@scriptis_appconn_id,'5','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@scriptis_appconn_id,'6','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@scriptis_appconn_id,'7','1',now(),'system'); + +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@workflow_appconn_id,'1','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@workflow_appconn_id,'2','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@workflow_appconn_id,'3','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@workflow_appconn_id,'4','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@workflow_appconn_id,'5','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@workflow_appconn_id,'6','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('-1',@workflow_appconn_id,'7','1',now(),'system'); + +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('224',@scriptis_appconn_id,'1','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('224',@workflow_appconn_id,'1','1',now(),'system'); +INSERT INTO `dss_workspace_appconn_role` (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) VALUES('224',@apiservice_appconn_id,'1','1',now(),'system'); + + +INSERT INTO `dss_workspace_admin_dept` (`id`, `parent_id`, `ancestors`, `dept_name`, `order_num`, `leader`, `phone`, `email`, `status`, `del_flag`, `create_by`, `create_time`, `update_by`, `update_time`) VALUES('100','0','0','基础科技','0','leader01','1888888888','123@qq.com','0','0','admin',now(),'admin',now()); diff --git a/db/version_update/from_v101_to_v110.sql b/db/version_update/from_v101_to_v110.sql new file mode 100644 index 000000000..83fa84976 --- /dev/null +++ b/db/version_update/from_v101_to_v110.sql @@ -0,0 +1,394 @@ +--- v1.0.7 +CREATE TABLE `dss_workflow_execute_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_id` bigint(20) NOT NULL COMMENT '任务id', + `status` int(1) DEFAULT NULL COMMENT '状态,0:失败 1:成功,', + `flow_id` bigint(20) NOT NULL COMMENT 'flowId', + `version` varchar(200) DEFAULT NULL COMMENT '工作流bml版本号', + `failed_jobs` text COMMENT '执行失败节点', + `Pending_jobs` text COMMENT '未执行节点', + `skipped_jobs` text COMMENT '执行跳过节点', + `succeed_jobs` text COMMENT '执行成功节点', + `createtime` datetime NOT NULL COMMENT '创建时间', + `running_jobs` text COMMENT '正在执行节点', + `updatetime` datetime DEFAULT NULL COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- 需要先添加一条记录,先删除原来spark节点内存校验的关联关系 +INSERT INTO `dss_workflow_node_ui_validate` (`validate_type`, `validate_range`, `error_msg`, `error_msg_en`, `trigger`) VALUES ('Regex', '^[0-9.]*g{0,1}$', 'Spark内存设置如2g', 'Drive memory size, default value: 2', 'blur'); + + CREATE TABLE `dss_orchestrator_job_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `job_id` bigint(20) DEFAULT NULL COMMENT 'job ID', + `conversion_job_json` text, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `updated_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='dss_orchestrator_job_info表'; + +-------------------- v1.1.2 + +DROP TABLE IF EXISTS `dss_admin_dept`; +CREATE TABLE `dss_admin_dept` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '部门id', + `parent_id` bigint(20) DEFAULT '0' COMMENT '父部门id', + `ancestors` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '祖级列表', + `dept_name` varchar(30) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '部门名称', + `order_num` int(4) DEFAULT '0' COMMENT '显示顺序', + `leader` varchar(20) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '负责人', + `phone` varchar(11) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '联系电话', + `email` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '邮箱', + `status` char(1) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '0' COMMENT '部门状态(0正常 1停用)', + `del_flag` char(1) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '0' COMMENT '删除标志(0代表存在 2代表删除)', + `create_by` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='部门表'; + +DROP TABLE IF EXISTS `dss_guide_group`; +CREATE TABLE IF NOT EXISTS `dss_guide_group` ( + `id` BIGINT(13) NOT NULL AUTO_INCREMENT, + `path` VARCHAR(100) NOT NULL COMMENT '页面URL路径', + `title` VARCHAR(255) DEFAULT NULL COMMENT '标题', + `description` VARCHAR(200) DEFAULT NULL COMMENT '描述', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='用户向导页面'; + +DROP TABLE IF EXISTS `dss_guide_content`; +CREATE TABLE IF NOT EXISTS `dss_guide_content` ( + `id` BIGINT(13) NOT NULL AUTO_INCREMENT, + `group_id` BIGINT(50) NOT NULL COMMENT '所属页面ID', + `path` VARCHAR(100) NOT NULL COMMENT '所属页面URL路径', + `title` VARCHAR(255) DEFAULT NULL COMMENT '标题', + `title_alias` VARCHAR(50) DEFAULT NULL COMMENT '标题简称', + `seq` VARCHAR(20) DEFAULT NULL COMMENT '序号', + `type` INT(1) DEFAULT '1' COMMENT '类型: 1-步骤step,2-问题question', + `content` TEXT DEFAULT NULL COMMENT 'Markdown格式的内容', + `content_html` MEDIUMTEXT DEFAULT NULL COMMENT 'Markdown内容转化为HTML格式', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='用户向导页面内容详情'; + +DROP TABLE IF EXISTS `dss_download_audit`; +CREATE TABLE `dss_download_audit` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `creator` varchar(255) COMMENT '创建者', + `tenant` varchar(255) COMMENT '租户', + `path` varchar(255) COMMENT '文件路径', + `sql` varchar(3000) COMMENT '执行sql脚本', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE = INNODB DEFAULT CHARSET = utf8 COMMENT = '文件下载审计'; + +DROP TABLE IF EXISTS `dss_guide_catalog`; +CREATE TABLE IF NOT EXISTS `dss_guide_catalog` ( + `id` BIGINT(13) NOT NULL AUTO_INCREMENT, + `parent_id` BIGINT(13) NOT NULL COMMENT '父级目录ID,-1代表最顶级目录', + `title` VARCHAR(255) DEFAULT NULL COMMENT '标题', + `description` VARCHAR(200) DEFAULT NULL COMMENT '描述', + `create_by` VARCHAR(255) DEFAULT NULL COMMENT '创建者', + `create_time` DATETIME DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` VARCHAR(255) DEFAULT NULL COMMENT '更新者', + `update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` TINYINT(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='用户向导知识库目录'; + +DROP TABLE IF EXISTS `dss_guide_chapter`; +CREATE TABLE IF NOT EXISTS `dss_guide_chapter` ( + `id` BIGINT(13) NOT NULL AUTO_INCREMENT, + `catalog_id` BIGINT(13) NOT NULL COMMENT '目录ID', + `title` VARCHAR(255) DEFAULT NULL COMMENT '标题', + `title_alias` VARCHAR(50) DEFAULT NULL COMMENT '标题简称', + `content` TEXT DEFAULT NULL COMMENT 'Markdown格式的内容', + `content_html` MEDIUMTEXT DEFAULT NULL COMMENT 'Markdown转换为html内容', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='用户向导知识库文章'; + +DROP TABLE IF EXISTS `dss_proxy_user`; +CREATE TABLE `dss_proxy_user` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `username` varchar(64) DEFAULT NULL, + `proxy_user_name` varchar(64) DEFAULT NULL, + `create_by` varchar(64) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `remark` varchar(500) DEFAULT NULL COMMENT '备注', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=214 DEFAULT CHARSET=utf8; + +CREATE TABLE `dss_dataapi_config` ( + `id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT COMMENT '主键', + `workspace_id` BIGINT ( 20 ) NOT NULL COMMENT '工作空间id', + `api_name` VARCHAR ( 255 ) NOT NULL COMMENT 'API名称', + `api_path` VARCHAR ( 255 ) NOT NULL unique COMMENT 'API Path', + `group_id` BIGINT ( 20 ) NOT NULL COMMENT 'API组id', + `api_type` VARCHAR ( 20 ) NOT NULL COMMENT 'API类型:GUIDE-向导模式,SQL-脚本模式', + `protocol` VARCHAR ( 20 ) NOT NULL COMMENT 'Http协议', + `datasource_id` BIGINT ( 20 ) NOT NULL COMMENT '数据源id', + `datasource_name` VARCHAR ( 50 ) DEFAULT NULL COMMENT '数据源名称', + `datasource_type` VARCHAR ( 20 ) DEFAULT NULL COMMENT '数据源类型', + `sql` text COMMENT 'sql模板', + `tbl_name` VARCHAR ( 100 ) DEFAULT NULL COMMENT '数据表名称', + `req_fields` VARCHAR ( 1000 ) DEFAULT NULL COMMENT '请求字段名称', + `res_fields` VARCHAR ( 1000 ) DEFAULT NULL COMMENT '返回字段名称', + `order_fields` VARCHAR ( 500 ) DEFAULT NULL COMMENT '排序字段名称及方式', + `is_test` TINYINT ( 1 ) DEFAULT '0' COMMENT '是否测试成功:0未测试(默认),1测试成功', + `status` TINYINT ( 1 ) DEFAULT '0' COMMENT 'API状态:0未发布(默认),1已发布', + `previlege` VARCHAR ( 20 ) DEFAULT 'WORKSPACE' COMMENT 'WORKSPACE,PRIVATE', + `method` VARCHAR ( 20 ) DEFAULT NULL COMMENT 'HTTPS,HTTP', + `describe` VARCHAR ( 255 ) DEFAULT NULL COMMENT '描述', + `memory` INT DEFAULT NULL COMMENT '内存大小', + `req_timeout` INT DEFAULT NULL COMMENT '请求超时时间', + `label` VARCHAR ( 255 ) DEFAULT NULL COMMENT '标签', + `create_by` VARCHAR ( 255 ) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` VARCHAR ( 255 ) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `is_delete` TINYINT ( 1 ) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + `page_size` int DEFAULT 0 COMMENT '每页数据大小', + PRIMARY KEY ( `id` ) +) ENGINE = INNODB DEFAULT CHARSET = utf8 COMMENT = 'API'; + +DROP TABLE IF EXISTS `dss_dataapi_group`; +CREATE TABLE `dss_dataapi_group` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL COMMENT '工作空间id', + `name` varchar(255) NOT NULL COMMENT '名称', + `note` varchar(255) DEFAULT NULL COMMENT '描述', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE = INNODB DEFAULT CHARSET = utf8 COMMENT='服务组'; + +DROP TABLE IF EXISTS `dss_dataapi_auth`; +CREATE TABLE `dss_dataapi_auth` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) NOT NULL COMMENT '工作空间ID', + `caller` varchar(255) DEFAULT NULL COMMENT '调用者名称', + `token` varchar(255) DEFAULT NULL COMMENT 'token字符串', + `expire` datetime DEFAULT NULL COMMENT 'token过期时间', + `group_id` bigint(20) DEFAULT NULL COMMENT 'api组', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='API认证'; + +DROP TABLE IF EXISTS `dss_dataapi_call`; +CREATE TABLE `dss_dataapi_call` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `api_id` bigint(11) NOT NULL COMMENT 'API ID', + `params_value` text COMMENT '调用参数名称和值', + `status` tinyint(255) DEFAULT NULL COMMENT '执行状态:1成功,2失败,3超时', + `time_start` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '开始时间', + `time_end` datetime DEFAULT NULL COMMENT '结束时间', + `time_length` bigint(20) DEFAULT NULL COMMENT '调用时长', + `caller` varchar(255) DEFAULT NULL COMMENT '调用者名称', + PRIMARY KEY (`id`) +) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='API调用记录' +; +DROP TABLE IF EXISTS `dss_dataapi_datasource`; +CREATE TABLE `dss_dataapi_datasource` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL COMMENT '工作空间id', + `name` varchar(255) NOT NULL COMMENT '名称', + `note` varchar(255) DEFAULT NULL COMMENT '描述', + `type` varchar(255) NOT NULL COMMENT '数据库类型', + `url` varchar(255) NOT NULL COMMENT '连接url', + `username` varchar(255) NOT NULL COMMENT '用户名', + `pwd` varchar(255) NOT NULL COMMENT '密码', + `create_by` varchar(255) DEFAULT NULL COMMENT '创建者', + `create_time` datetime DEFAULT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(255) DEFAULT NULL COMMENT '更新者', + `update_time` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `is_delete` tinyint(1) DEFAULT '0' COMMENT '0:未删除(默认), 1已删除', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='数据源'; + + +alter table dss_onestop_user_favorites add `type` varchar(20) DEFAULT NULL COMMENT '类型,#区分收藏和盯一盯'; +ALTER TABLE dss_orchestrator_info modify column uuid varchar(100) not null; +alter table dss_workspace add `workspace_type` varchar(20); + +alter table dss_user add `dept_id` int(11) DEFAULT NULL; +alter table dss_user add `email` varchar(50) DEFAULT '' COMMENT '用户邮箱'; +alter table dss_user add `phonenumber` varchar(11) DEFAULT '' COMMENT '手机号码'; +alter table dss_user add `password` varchar(100) DEFAULT '' COMMENT '密码'; +alter table dss_user add `del_flag` char(1) DEFAULT '0' COMMENT '删除标志(0代表存在 2代表删除)'; +alter table dss_user add `create_by` varchar(64) DEFAULT NULL COMMENT '创建者'; +alter table dss_user add `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间'; +alter table dss_user add `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP; +alter table dss_user add `remark` varchar(500) DEFAULT NULL COMMENT '备注'; + +ALTER TABLE dss_flow_edit_lock MODIFY COLUMN flow_version varchar(16) NULL; + +------------------- v1.1.3 +-- todo 最好先把dss库备份一份,方便查询原始数据, dss_appconn表的id改为从1001开始,避免太小和之前application_id混淆 + +-- todo appconn_id测试环境不超过20,生产需要确认 +INSERT INTO dss_appconn +(id, appconn_name, is_user_need_init, `level`, if_iframe, is_external, reference, class_name, appconn_class_path, resource) +VALUES(22, 'scriptis', 0, 1, 0, 0, 'sso', NULL, NULL, ''); +INSERT INTO dss_appconn +(id, appconn_name, is_user_need_init, `level`, if_iframe, is_external, reference, class_name, appconn_class_path, resource) +VALUES(23, 'sso', 0, 1, 0, 0, NULL, 'com.webank.wedatasphere.dss.appconn.sso.SSOAppConn', '', ''); +INSERT INTO dss_appconn +(id, appconn_name, is_user_need_init, `level`, if_iframe, is_external, reference, class_name, appconn_class_path, resource) +VALUES(24, 'apiservice', 0, 1, 0, 0, 'sso', NULL, NULL, ''); + +-- TODO dss_appconn_instance修改 +alter table dss_appconn_instance drop column redirect_url; +alter table dss_appconn_instance change column homepage_url `homepage_uri` varchar(255) DEFAULT NULL COMMENT '主页uri'; + +INSERT INTO dss_appconn_instance +(appconn_id, label, url, enhance_json, homepage_uri) +VALUES(22, 'DEV', '/home', '', ''); +INSERT INTO dss_appconn_instance +(appconn_id, label, url, enhance_json, homepage_uri) +VALUES(24, 'DEV', '/apiservices', '', ''); + +update dss_appconn set id=id+1000; +update dss_appconn_instance set appconn_id=appconn_id+1000; +-- orchestrator-framework没用到了,需删除。 +delete from dss_appconn where appconn_name="orchestrator-framework"; +update dss_appconn set class_name="com.webank.wedatasphere.dss.appconn.workflow.WorkflowAppConn" where appconn_name="workflow"; +-- todo if_iframe和is_external字段修改,先都设置为0,再逐个update +update dss_appconn set if_iframe =0,is_external =0; +update dss_appconn set if_iframe=1,is_external=1 where appconn_name="schedulis"; +update dss_appconn set if_iframe=1,is_external=1 where appconn_name="visualis"; +update dss_appconn set if_iframe=1,is_external=0 where appconn_name="apiservice"; + +alter table dss_orchestrator_info add column (`workspace_id` int(11) DEFAULT NULL COMMENT '空间id', + `orchestrator_mode` varchar(100) DEFAULT NULL COMMENT '编排模式,取得的值是dss_dictionary中的dic_key(parent_key=p_arrangement_mode)', + `orchestrator_way` varchar(256) DEFAULT NULL COMMENT '编排方式', + `orchestrator_level` varchar(32) DEFAULT NULL, + `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间'); + +update dss_orchestrator_info oi set workspace_id=(select workspace_id from dss_project_orchestrator po where po.orchestrator_id=oi.id); +update dss_orchestrator_info oi set orchestrator_mode=(select orchestrator_mode from dss_project_orchestrator po where po.orchestrator_id=oi.id); +update dss_orchestrator_info oi set orchestrator_way=(select orchestrator_way from dss_project_orchestrator po where po.orchestrator_id=oi.id); +update dss_orchestrator_info oi set update_user=(select update_user from dss_project_orchestrator po where po.orchestrator_id=oi.id); +update dss_orchestrator_info oi set update_time=(select update_time from dss_project_orchestrator po where po.orchestrator_id=oi.id); + +-- dss_orchestrator_ref_orchestration_relation definition +CREATE TABLE `dss_orchestrator_ref_orchestration_relation` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `orchestrator_id` bigint(20) NOT NULL, + `ref_project_id` bigint(20) DEFAULT NULL, + `ref_orchestration_id` bigint(20) DEFAULT NULL COMMENT '调度系统工作流的id(调用SchedulerAppConn的OrchestrationOperation服务返回的orchestrationId)', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + + +ALTER TABLE dss_workflow_node CHANGE icon icon_path longtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL; +-- todo 设置每个节点的jump_type和appconn_name,jump_url要先改为1,否则改变不了类型为int +update dss_workflow_node set jump_url ="1"; +ALTER TABLE dss_workflow_node CHANGE jump_url jump_type INT NULL; +update dss_workflow_node set icon_path =concat("icons/",name,".icon"); +-- todo 确认生产各个节点support_jump字段是否是正确的 +update dss_workflow_node set appconn_name="scriptis",jump_type=2 where name in ('python','pyspark','sql','shell','scala','hql','jdbc','connector','subFlow'); +update dss_workflow_node set appconn_name="visualis" where name in ('display','dashboard','widget','view'); +update dss_workflow_node set appconn_name="sendemail" where name in ('sendemail'); +update dss_workflow_node set appconn_name="eventchecker" where name in ('eventsender','eventreceiver'); +update dss_workflow_node set appconn_name="datachecker" where name in ('datachecker'); +update dss_workflow_node set jump_type=0 where name in ("eventsender","eventreceiver","datachecker"); + +RENAME TABLE dss_dictionary to dss_workspace_dictionary; +-- todo 检查下dss_workspace_role是否有存量数据 +DROP TABLE IF EXISTS `dss_workspace_role`; +RENAME TABLE dss_role to dss_workspace_role; +RENAME TABLE dss_admin_dept to dss_workspace_admin_dept; +RENAME TABLE dss_download_audit to dss_workspace_download_audit; +RENAME TABLE dss_flow_relation to dss_workflow_relation; +RENAME TABLE dss_flow_edit_lock to dss_workflow_edit_lock; +RENAME TABLE dss_onestop_menu to dss_workspace_menu; +RENAME TABLE dss_menu_role to dss_workspace_menu_role; + +RENAME TABLE dss_onestop_user_favorites to dss_workspace_user_favorites_appconn; +-- todo application_id和appconn_id对应关系修改,通过dss_onestop_menu_application的application_id到dss_application表找到组件,待确认生产数据 +ALTER TABLE dss_workspace_user_favorites_appconn CHANGE menu_application_id menu_appconn_id int(20) NULL; + +-- todo application_id(dss_application表)和appconn_id对应关系修改 +RENAME TABLE dss_component_role to dss_workspace_appconn_role; +ALTER TABLE dss_workspace_appconn_role CHANGE component_id appconn_id int(20) NULL; +-- dss_application的linkis组件相当于scriptis +select @linkis_application_id:=id from dss_application where name='linkis'; +select @scriptis_appconn_id:=id from dss_appconn where appconn_name='scriptis'; +update dss_workspace_appconn_role set appconn_id=@scriptis_appconn_id where appconn_id=@linkis_application_id; + +select @workflow_application_id:=id from dss_application where name='workflow'; +select @workflow_appconn_id:=id from dss_appconn where appconn_name='workflow'; +update dss_workspace_appconn_role set appconn_id=@workflow_appconn_id where appconn_id=@workflow_application_id; + +select @visualis_application_id:=id from dss_application where name='visualis'; +select @visualis_appconn_id:=id from dss_appconn where appconn_name='visualis'; +update dss_workspace_appconn_role set appconn_id=@visualis_appconn_id where appconn_id=@visualis_application_id; + +select @schedulis_application_id:=id from dss_application where name='schedulis'; +select @schedulis_appconn_id:=id from dss_appconn where appconn_name='schedulis'; +update dss_workspace_appconn_role set appconn_id=@schedulis_appconn_id where appconn_id=@schedulis_application_id; + +select @qualitis_application_id:=id from dss_application where name='qualitis'; +select @qualitis_appconn_id:=id from dss_appconn where appconn_name='qualitis'; +update dss_workspace_appconn_role set appconn_id=@qualitis_appconn_id where appconn_id=@qualitis_application_id; + +-- todo 确定生产apiservice的name +select @apiservice_application_id:=id from dss_application where name='apiService'; +select @apiservice_appconn_id:=id from dss_appconn where appconn_name='apiservice'; +update dss_workspace_appconn_role set appconn_id=@apiservice_appconn_id where appconn_id=@apiservice_application_id; + + +-- 和workspace_menu表可以dml语句插入,todo appconn_id手动修改 +RENAME TABLE dss_onestop_menu_application to dss_workspace_menu_appconn; +ALTER TABLE dss_workspace_menu_appconn CHANGE application_id appconn_id int(20) NULL; +ALTER TABLE dss_workspace_menu_appconn CHANGE onestop_menu_id menu_id int(20) NULL; +-- todo 确定title_en是否正确 +select @apiservice_appconn_id:=id from dss_appconn where appconn_name='apiservice'; +update dss_workspace_menu_appconn set appconn_id=@apiservice_appconn_id where title_en="Data service development"; + +select @scriptis_appconn_id:=id from dss_appconn where appconn_name='scriptis'; +update dss_workspace_menu_appconn set appconn_id=@scriptis_appconn_id where title_en="Scriptis"; + +select @workflow_appconn_id:=id from dss_appconn where appconn_name='workflow'; +update dss_workspace_menu_appconn set appconn_id=@workflow_appconn_id where title_en="workflow"; + +select @visualis_appconn_id:=id from dss_appconn where appconn_name='visualis'; +update dss_workspace_menu_appconn set appconn_id=@visualis_appconn_id where title_en="Visualis"; + +select @schedulis_appconn_id:=id from dss_appconn where appconn_name='schedulis'; +update dss_workspace_menu_appconn set appconn_id=@schedulis_appconn_id where title_en="Schedulis"; + + +-- v1.1.4 修正datachecker类名 +update dss_appconn set class_name="com.webank.wedatasphere.dss.appconn.datachecker.DataCheckerAppConn" where appconn_name="datachecker"; + + + + + + diff --git a/db/version_update/from_v110_to_v111.sql b/db/version_update/from_v110_to_v111.sql new file mode 100644 index 000000000..a78764a78 --- /dev/null +++ b/db/version_update/from_v110_to_v111.sql @@ -0,0 +1,8 @@ +select @schedulis_appconn_id:=id from dss_appconn where appconn_name='schedulis'; +update dss_appconn_instance set enhance_json = '{"reqUri":""}', homepage_uri = '/manager' where appconn_id = @schedulis_appconn_id; + +select @qualitis_appconn_id:=id from dss_appconn where appconn_name='qualitis'; +update dss_appconn_instance set enhance_json = '{"reqUri":""}', homepage_uri = '#/dashboard' where appconn_id = @qualitis_appconn_id; + +select @dolphinscheduler_appconn_id:=id from dss_appconn where appconn_name='dolphinscheduler'; +update dss_appconn_instance set enhance_json = '{"reqUri":"dolphinscheduler/projects/analysis/define-user-count"}', homepage_uri = 'dolphinscheduler' where appconn_id = @dolphinscheduler_appconn_id; \ No newline at end of file diff --git a/db/wait_for_change/changing_to_dss_ddl.sql b/db/wait_for_change/changing_to_dss_ddl.sql new file mode 100644 index 000000000..c71cf994b --- /dev/null +++ b/db/wait_for_change/changing_to_dss_ddl.sql @@ -0,0 +1,212 @@ + +DROP TABLE IF EXISTS `dss_workspace_appconn_role`; -- changed +CREATE TABLE `dss_workspace_appconn_role` ( -- changed + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `appconn_id` int(20) DEFAULT NULL, -- changed + `role_id` int(20) DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `updateby` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=5103 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_workflow_edit_lock`; -- changed +CREATE TABLE `dss_workflow_edit_lock` ( -- changed + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '', + `workflow_id` bigint(11) NOT NULL COMMENT '', -- changed + `flow_version` varchar(16) NOT NULL COMMENT '', -- delete it + `username` varchar(64) NOT NULL COMMENT '', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `owner` varchar(128) NOT NULL COMMENT '', + `lock_stamp` int(8) NOT NULL DEFAULT '0' COMMENT '', -- delete it + `is_expire` tinyint(1) NOT NULL DEFAULT '0' COMMENT '', + `lock_content` varchar(512) NOT NULL COMMENT '', + PRIMARY KEY (`id`), + UNIQUE KEY `dss_flow_edit_lock_flow_id_IDX` (`flow_id`) USING BTREE + ) ENGINE=InnoDB AUTO_INCREMENT=571 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_workspace_menu`; -- changed +CREATE TABLE `dss_workspace_menu` ( -- changed + `id` int(20) NOT NULL AUTO_INCREMENT, + `name` varchar(64) DEFAULT NULL, + `title_en` varchar(64) DEFAULT NULL, + `title_cn` varchar(64) DEFAULT NULL, + `description` varchar(255) DEFAULT NULL, + `is_active` tinyint(1) DEFAULT '1', + `icon` varchar(255) DEFAULT NULL, + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_workspace_menu_role`; -- changed +CREATE TABLE `dss_workspace_menu_role` ( -- changed + `id` int(20) NOT NULL AUTO_INCREMENT, + `workspace_id` int(20) DEFAULT NULL, + `menu_id` int(20) DEFAULT NULL, -- changed + `role_id` int(20) DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `updateby` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=5263 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_workspace_menu_appconn`; -- changed +CREATE TABLE `dss_workspace_menu_appconn` ( -- changed + `id` int(20) NOT NULL AUTO_INCREMENT, + `appconn_id` int(20) DEFAULT NULL, -- changed + `menu_id` int(20) NOT NULL, -- changed + `title_en` varchar(64) DEFAULT NULL, + `title_cn` varchar(64) DEFAULT NULL, + `desc_en` varchar(255) DEFAULT NULL, + `desc_cn` varchar(255) DEFAULT NULL, + `labels_en` varchar(255) DEFAULT NULL, + `labels_cn` varchar(255) DEFAULT NULL, + `is_active` tinyint(1) DEFAULT NULL, + `access_button_en` varchar(64) DEFAULT NULL, + `access_button_cn` varchar(64) DEFAULT NULL, + `manual_button_en` varchar(64) DEFAULT NULL, + `manual_button_cn` varchar(64) DEFAULT NULL, + `manual_button_url` varchar(255) DEFAULT NULL, + `icon` varchar(255) DEFAULT NULL, + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + `image` varchar(200) DEFAULT NULL COMMENT '图片', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_workspace_user_favorites_appconn`; -- changed +CREATE TABLE `dss_workspace_user_favorites_appconn` ( -- changed + `id` int(20) NOT NULL AUTO_INCREMENT, + `username` varchar(64) DEFAULT NULL, + `workspace_id` bigint(20) DEFAULT '1', + `menu_appconn_id` int(20) DEFAULT NULL, -- changed + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + `type` varchar(20) NOT NULL DEFAULT "" COMMENT "dingyiding or 收藏", + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=94 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_workspace_role`; -- changed +CREATE TABLE `dss_workspace_role` ( -- changed + `id` int(20) NOT NULL AUTO_INCREMENT, + `workspace_id` varchar(255) DEFAULT NULL, + `name` varchar(255) DEFAULT NULL, + `front_name` varchar(255) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `description` varchar(512) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `workspace_id` (`workspace_id`,`name`) +) ENGINE=InnoDB AUTO_INCREMENT=10 DEFAULT CHARSET=utf8 CHECKSUM=1 DELAY_KEY_WRITE=1 ROW_FORMAT=DYNAMIC; + + +DROP TABLE IF EXISTS `dss_workflow_relation`; -- changed +CREATE TABLE `dss_workflow_relation` ( -- changed + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `flow_id` bigint(20) DEFAULT NULL, + `parent_flow_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=78 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + + +DROP TABLE IF EXISTS `dss_workspace_admin_dept`; -- changed +CREATE TABLE `dss_workspace_admin_dept` ( -- changed + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '部门id', + `parent_id` bigint(20) DEFAULT '0' COMMENT '父部门id', + `ancestors` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '祖级列表', + `dept_name` varchar(30) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '部门名称', + `order_num` int(4) DEFAULT '0' COMMENT '显示顺序', + `leader` varchar(20) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '负责人', + `phone` varchar(11) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '联系电话', + `email` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '邮箱', + `status` char(1) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '0' COMMENT '部门状态(0正常 1停用)', + `del_flag` char(1) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '0' COMMENT '删除标志(0代表存在 2代表删除)', + `create_by` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='部门表'; + + +DROP TABLE IF EXISTS `dss_workspace_user`; -- delete this table +CREATE TABLE `dss_workspace_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `username` varchar(32) DEFAULT NULL, + `join_time` datetime DEFAULT NULL, + `created_by` varchar(255) DEFAULT NULL, + `user_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `workspace_id` (`workspace_id`,`username`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 comment '空间用户表'; + +DROP TABLE IF EXISTS `dss_workspace_user_role`; -- use this table +CREATE TABLE `dss_workspace_user_role` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `username` varchar(32) DEFAULT NULL, + `role_id` int(20) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `created_by` varchar(255) DEFAULT NULL, + `user_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 comment '空间用户角色关系表'; + + +DROP TABLE IF EXISTS `dss_workspace_download_audit`; -- changed +CREATE TABLE `dss_workspace_download_audit` ( -- changed + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `creator` varchar(255) COMMENT '创建者', + `tenant` varchar(255) COMMENT '租户', + `path` varchar(255) COMMENT '文件路径', + `sql` varchar(3000) COMMENT '执行sql脚本', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE = INNODB DEFAULT CHARSET = utf8 COMMENT = '文件下载审计'; + + +DROP TABLE IF EXISTS `dss_workspace_dictionary`; -- changed +CREATE TABLE `dss_workspace_dictionary` ( -- changed + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `workspace_id` int(11) DEFAULT '0' COMMENT '空间ID,默认为0,所有空间都有', + `parent_key` varchar(200) DEFAULT '0' COMMENT '父key', + `dic_name` varchar(200) NOT NULL COMMENT '名称', + `dic_name_en` varchar(300) DEFAULT NULL COMMENT '名称(英文)', + `dic_key` varchar(200) NOT NULL COMMENT 'key 相当于编码,空间是w_开头,工程是p_', + `dic_value` varchar(500) DEFAULT NULL COMMENT 'key对应的值', + `dic_value_en` varchar(1000) DEFAULT NULL COMMENT 'key对应的值(英文)', + `title` varchar(200) DEFAULT NULL COMMENT '标题', + `title_en` varchar(400) DEFAULT NULL COMMENT '标题(英文)', + `url` varchar(200) DEFAULT NULL COMMENT 'url', + `url_type` int(1) DEFAULT '0' COMMENT 'url类型: 0-内部系统,1-外部系统;默认是内部', + `icon` varchar(200) DEFAULT NULL COMMENT '图标', + `order_num` int(2) DEFAULT '1' COMMENT '序号', + `remark` varchar(1000) DEFAULT NULL COMMENT '备注', + `create_user` varchar(100) DEFAULT NULL COMMENT '创建人', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `idx_unique_workspace_id` (`workspace_id`,`dic_key`), + KEY `idx_parent_key` (`parent_key`), + KEY `idx_dic_key` (`dic_key`) +) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8 COMMENT='数据字典表'; \ No newline at end of file diff --git a/db/wait_for_change/origin_dss_ddl.sql b/db/wait_for_change/origin_dss_ddl.sql new file mode 100644 index 000000000..39083e5bc --- /dev/null +++ b/db/wait_for_change/origin_dss_ddl.sql @@ -0,0 +1,221 @@ +DROP TABLE IF EXISTS `dss_component_role`; +CREATE TABLE `dss_component_role` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `component_id` int(20) DEFAULT NULL, -- 其实是 application_id + `role_id` int(20) DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `updateby` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=5103 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_flow_edit_lock`; +CREATE TABLE `dss_flow_edit_lock` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '', + `flow_id` bigint(11) NOT NULL COMMENT '', + `flow_version` varchar(16) NOT NULL COMMENT '', + `username` varchar(64) NOT NULL COMMENT '', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `owner` varchar(128) NOT NULL COMMENT '', + `lock_stamp` int(8) NOT NULL DEFAULT '0' COMMENT '', + `is_expire` tinyint(1) NOT NULL DEFAULT '0' COMMENT '', + `lock_content` varchar(512) NOT NULL COMMENT '', + PRIMARY KEY (`id`), + UNIQUE KEY `dss_flow_edit_lock_flow_id_IDX` (`flow_id`) USING BTREE + ) ENGINE=InnoDB AUTO_INCREMENT=571 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_onestop_menu`; +CREATE TABLE `dss_onestop_menu` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `name` varchar(64) DEFAULT NULL, + `title_en` varchar(64) DEFAULT NULL, + `title_cn` varchar(64) DEFAULT NULL, + `description` varchar(255) DEFAULT NULL, + `is_active` tinyint(1) DEFAULT '1', + `icon` varchar(255) DEFAULT NULL, + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_menu_role`; +CREATE TABLE `dss_menu_role` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `workspace_id` int(20) DEFAULT NULL, + `menu_id` int(20) DEFAULT NULL, + `role_id` int(20) DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `updateby` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=5263 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_onestop_menu_application`; +CREATE TABLE `dss_onestop_menu_application` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `application_id` int(20) DEFAULT NULL, + `onestop_menu_id` int(20) NOT NULL, + `title_en` varchar(64) DEFAULT NULL, + `title_cn` varchar(64) DEFAULT NULL, + `desc_en` varchar(255) DEFAULT NULL, + `desc_cn` varchar(255) DEFAULT NULL, + `labels_en` varchar(255) DEFAULT NULL, + `labels_cn` varchar(255) DEFAULT NULL, + `is_active` tinyint(1) DEFAULT NULL, + `access_button_en` varchar(64) DEFAULT NULL, + `access_button_cn` varchar(64) DEFAULT NULL, + `manual_button_en` varchar(64) DEFAULT NULL, + `manual_button_cn` varchar(64) DEFAULT NULL, + `manual_button_url` varchar(255) DEFAULT NULL, + `icon` varchar(255) DEFAULT NULL, + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + `image` varchar(200) DEFAULT NULL COMMENT '图片', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_onestop_user_favorites`; +CREATE TABLE `dss_onestop_user_favorites` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `username` varchar(64) DEFAULT NULL, + `workspace_id` bigint(20) DEFAULT '1', + `menu_application_id` int(20) DEFAULT NULL, + `order` int(2) DEFAULT NULL, + `create_by` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `last_update_user` varchar(30) DEFAULT NULL, + `type` varchar(20) NOT NULL DEFAULT "" COMMENT "dingyiding or 收藏", + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=94 DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `dss_role`; +CREATE TABLE `dss_role` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `workspace_id` varchar(255) DEFAULT NULL, + `name` varchar(255) DEFAULT NULL, + `front_name` varchar(255) DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `description` varchar(512) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `workspace_id` (`workspace_id`,`name`) +) ENGINE=InnoDB AUTO_INCREMENT=10 DEFAULT CHARSET=utf8 CHECKSUM=1 DELAY_KEY_WRITE=1 ROW_FORMAT=DYNAMIC; + + +DROP TABLE IF EXISTS `dss_flow_relation`; +CREATE TABLE `dss_flow_relation` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `flow_id` bigint(20) DEFAULT NULL, + `parent_flow_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=78 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; + + +DROP TABLE IF EXISTS `dss_flow_edit_lock`; +CREATE TABLE `dss_flow_edit_lock` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '', + `flow_id` bigint(11) NOT NULL COMMENT '', + `flow_version` varchar(16) NOT NULL COMMENT '', + `username` varchar(64) NOT NULL COMMENT '', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `owner` varchar(128) NOT NULL COMMENT '', + `lock_stamp` int(8) NOT NULL DEFAULT '0' COMMENT '', + `is_expire` tinyint(1) NOT NULL DEFAULT '0' COMMENT '', + `lock_content` varchar(512) NOT NULL COMMENT '', + PRIMARY KEY (`id`), + UNIQUE KEY `dss_flow_edit_lock_flow_id_IDX` (`flow_id`) USING BTREE + ) ENGINE=InnoDB AUTO_INCREMENT=571 DEFAULT CHARSET=utf8; + + +DROP TABLE IF EXISTS `dss_admin_dept`; +CREATE TABLE `dss_admin_dept` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '部门id', + `parent_id` bigint(20) DEFAULT '0' COMMENT '父部门id', + `ancestors` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '祖级列表', + `dept_name` varchar(30) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '部门名称', + `order_num` int(4) DEFAULT '0' COMMENT '显示顺序', + `leader` varchar(20) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '负责人', + `phone` varchar(11) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '联系电话', + `email` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '邮箱', + `status` char(1) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '0' COMMENT '部门状态(0正常 1停用)', + `del_flag` char(1) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '0' COMMENT '删除标志(0代表存在 2代表删除)', + `create_by` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '创建者', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '更新者', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='部门表'; + + +DROP TABLE IF EXISTS `dss_workspace_user`; +CREATE TABLE `dss_workspace_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `username` varchar(32) DEFAULT NULL, + `join_time` datetime DEFAULT NULL, + `created_by` varchar(255) DEFAULT NULL, + `user_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `workspace_id` (`workspace_id`,`username`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 comment '空间用户表'; +DROP TABLE IF EXISTS `dss_workspace_user_role`; +CREATE TABLE `dss_workspace_user_role` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `username` varchar(32) DEFAULT NULL, + `role_id` int(20) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `created_by` varchar(255) DEFAULT NULL, + `user_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 comment '空间用户角色关系表'; + + +DROP TABLE IF EXISTS `dss_download_audit`; +CREATE TABLE `dss_download_audit` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `creator` varchar(255) COMMENT '创建者', + `tenant` varchar(255) COMMENT '租户', + `path` varchar(255) COMMENT '文件路径', + `sql` varchar(3000) COMMENT '执行sql脚本', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE = INNODB DEFAULT CHARSET = utf8 COMMENT = '文件下载审计'; + + +DROP TABLE IF EXISTS `dss_dictionary`; +CREATE TABLE `dss_dictionary` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `workspace_id` int(11) DEFAULT '0' COMMENT '空间ID,默认为0,所有空间都有', + `parent_key` varchar(200) DEFAULT '0' COMMENT '父key', + `dic_name` varchar(200) NOT NULL COMMENT '名称', + `dic_name_en` varchar(300) DEFAULT NULL COMMENT '名称(英文)', + `dic_key` varchar(200) NOT NULL COMMENT 'key 相当于编码,空间是w_开头,工程是p_', + `dic_value` varchar(500) DEFAULT NULL COMMENT 'key对应的值', + `dic_value_en` varchar(1000) DEFAULT NULL COMMENT 'key对应的值(英文)', + `title` varchar(200) DEFAULT NULL COMMENT '标题', + `title_en` varchar(400) DEFAULT NULL COMMENT '标题(英文)', + `url` varchar(200) DEFAULT NULL COMMENT 'url', + `url_type` int(1) DEFAULT '0' COMMENT 'url类型: 0-内部系统,1-外部系统;默认是内部', + `icon` varchar(200) DEFAULT NULL COMMENT '图标', + `order_num` int(2) DEFAULT '1' COMMENT '序号', + `remark` varchar(1000) DEFAULT NULL COMMENT '备注', + `create_user` varchar(100) DEFAULT NULL COMMENT '创建人', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `idx_unique_workspace_id` (`workspace_id`,`dic_key`), + KEY `idx_parent_key` (`parent_key`), + KEY `idx_dic_key` (`dic_key`) +) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8 COMMENT='数据字典表'; \ No newline at end of file diff --git a/docs/zh_CN/ch3/DSS_User_Manual.md b/docs/zh_CN/ch3/DSS_User_Manual.md deleted file mode 100644 index e5c641803..000000000 --- a/docs/zh_CN/ch3/DSS_User_Manual.md +++ /dev/null @@ -1,221 +0,0 @@ -## 1 功能简介 -       DSS作为一站式数据应用开发门户,定位为闭环涵盖数据应用的全流程,满足从数据ETL、数据研发、可视化展现、数据治理、数据输出到工作流调度的数据应用全生命周期开发场景,现已经开源的组件包括如下图所示: - -![functions](/images/zh_CN/charpter3/manual/functions.png) - -- 工作流服务——Workflow: 支持工作流的托拉拽生成,支持实时执行,支持发布到调度服务 -- 数据研发———Scriptis:为DSS集成数据开发能力,支持各类型脚本语言的在线开发,详细介绍请点击[Scriptis使用手册]() -- 数据可视化——Visualis: 提供可视化能力,允许数据大屏和仪表盘作为DSS应用开发节点,详细介绍请点击[Visualis使用手册](); -- 数据发送——SendEmail: 为DSS集成数据发送能力,支持table,DashBoard,文本等的发送; -- 数据质量——Qualitis: 为DSS集成数据校验能力,将数据质量系统集成到DSS工作流开发中,对数据完整性、正确性等进行校验[Qualitis使用手册](); -- 信号节点:事件节点(发送和接收),用于信号的传递,需要配对使用。DataChecker:用于检查Hive中的表是否准备好,如果准备后则开始执行下面的流程; -- 功能节点:连接节点(空节点),用于做连接用,子工作流,用于工作流嵌套。 - -下面将对这些组件的功能进行详细介绍 - -## 2 DSS首页介绍 -### 2.1 已经集成组件展示: -       首页对DSS集成的组件的全流程进行了展示和示意,其中每个组件的展示都支持直接跳转到对应的服务: - -![components](/images/zh_CN/charpter3/manual/components.png) - -如点击Scriptis图标会跳转到数据开发组件Scriptis的首页: - -![components02](/images/zh_CN/charpter3/manual/components02.png) - - -### 2.2 DSS 工程: -       DSS为用户构建了工程业务流程两层结构,当用户选择一个工程进行进入后,在所有组件中的操作都只与该工程相关,让用户更加清晰的去定义工程和业务的关系。 -- 工程:您可以通过将一个产品对应到一个工程,用来涵盖该产品下的多个业务流程 -- 工作流:对应一个业务流程,让您在开发的过程中以业务的视角来组织和系统开发 - -1. 现阶段工程支持版本控制,发布等功能,后续会对协同开发进行支持: - -![project](/images/zh_CN/charpter3/manual/project.png) - -2. 工程版本新建(回滚):可以通过设置当前版本作为源版本复制为最新版本。 - -3. 工程复制:以工程的最新版本为源工程,复制出新工程,初始版本工作流内容为源工程最新版本的工作流。注意:**工程名是唯一,不可重复** - -## 3工作流——workflow -### 3.1 工作流编排 -       当点击一个对应的工程后,既可以进入工程首页,在工程首页可以做工作流的编排。 -1. 首先需要创建工作流 -![workflow01](/images/zh_CN/charpter3/manual/workflow01.png) -2. 建立好工作流好就可以进行组件节点的托拉拽了,支持第一章所提到的组件节点编排: -![workflow02](/images/zh_CN/charpter3/manual/workflow02.png) -3. 节点支持右键功能包括,删除、依赖选择、复制等基本功能,同时数据开发节点还支持脚本关联 -![workflow03](/images/zh_CN/charpter3/manual/workflow03.png) - -### 3.2 工作流节点打开 -节点支持双击打开: -1. 数据开发节点:点开后即可进入Scriptis进行脚本编辑 -![workflow04](/images/zh_CN/charpter3/manual/workflow04.png) -2. 子工作流节点:双击后跳转到子工作流编辑页面 -![workflow05](/images/zh_CN/charpter3/manual/workflow05.png) -3. 数据质量节点:跳转到数据质量规则配置页面 -![workflow06](/images/zh_CN/charpter3/manual/workflow06.png) -4. 数据可视化节点:跳转到对应的可视化编辑页面 -![workflow07](/images/zh_CN/charpter3/manual/workflow07.png) - -### 3.3 层级切换 -1. 支持多层级切换:支持快速工程切换、支持在工作流页面切换工作流、支持在单个工作流中切换节点 -![workflow08](/images/zh_CN/charpter3/manual/workflow08.png) - -2. 右上脚支持多组件快速切换,在切换后进入的组件的内容都只与该工程相关,让用户更加清晰的去定义工程和业务的内容: -![functions](/images/zh_CN/charpter3/manual/functions.png) - -### 3.4 参数和资源设置 - -1. 工作流上下文信息设置,支持工作流参数、变量、代理用户等 - -![workflow_09](/images/zh_CN/charpter3/manual/workflow_09.png) - -2. 支持工作流资源文件设置 -![workflow10](/images/zh_CN/charpter3/manual/workflow10.png) -支持工程级别、工作流级别、节点级别资源文件使用,您只需要在脚本中指定对应的级别就好:比如有一个test.txt 在脚本python脚本打开一个文件: -``` -open("project://test.txt", encoding="utf-8") #工程级资源文件使用project://开头 -open("flow://test.txt", encoding="utf-8") #工作流级资源文件使用flow://开头 -open("node://test.txt", encoding="utf-8") #节点级资源文件使用node://开头 -``` - -### 3.5 工作流实时执行 -1. 除了功能节点中的subflow会跳过执行,连接节点会作为空节点运行,其他都支持实时执行 -![workflow11](/images/zh_CN/charpter3/manual/workflow11.png) -2. 用户编辑好工作流后点击执行就可以将工作流进行运行,您将看到实时的工作流运行起来可以看到现在运行节点的时间,同时可以右键节点打开节点的管理台去展示该节点的进度,运行结果,运行日志等。支持任务停止等功能 -![workflow12](/images/zh_CN/charpter3/manual/workflow12.png) - - -### 3.6 工作流调度执行 -1. DSS的工程支持发布调度,默认支持发布到Azkaban,同样DSS的调度部分做了深层次的抽象可以做到对其他的调度系统快速支持。发布前会对工作流进行解析,以确保工作流是可以调度运行的: -![workflow13](/images/zh_CN/charpter3/manual/workflow13.png) -2. 发布后即可到调度系统中进行查看,比如去Azkaban页面上进行查看: -![workflow14](/images/zh_CN/charpter3/manual/workflow14.png) -3. DSS如何对接调度系统可以参考:[]() - -### 3.7 工作流版本 -1. 工作流创建完成后,具有初始版本,版本号为v000001,直接点击工作流图标时,默认打开工作流的最新版本 -2. 可以查看工作流的版本,方便您进行历史版本查看: -![workflow15](/images/zh_CN/charpter3/manual/workflow15.png) - -### 3.8 工作流布局修改 -1. 工作流格式化:当工作流节点过多,界面太乱时。可以点击节点编辑页的右上方第四个“格式化”按钮。快速美化节点界面: -![workflow16](/images/zh_CN/charpter3/manual/workflow16.png) -如果格式化后不满意,可再次点击节点编辑页的右上方第五个“恢复”按钮,恢复到之前的状态: -![workflow17](/images/zh_CN/charpter3/manual/workflow17.png) - -2. 支持放大、缩小、全屏/取消全屏 按钮可调整界面大小比例 - -3. 多个节点移动: - - 第一种方式是按住键盘“shift”+鼠标拖动,完全包括住的节点为选中节点,选中的节点有一圈灰色轮廓色。此时即可进行移动了 - - 第二种方式为按住键盘“ctrl”+鼠标点击选中的节点,选中的节点有一圈灰色轮廓色。此时即可进行移动了 -![workflow18](/images/zh_CN/charpter3/manual/workflow18.png) - - - -## 4 信号节点的使用: - -### 4.1 eventSender节点: -       EventSender节点用于进行信息发送,将一段信息事件进行发送给eventReceiver。常见场景工程与工程间有依赖,工作流与工作流间有信息依赖。比如B任务依赖A任务的某些信息(A任务成功B节点才能开始执行),eventSender支持如下参数: -``` -1. msg.type: 用来指定Job的类型,SEND用于发送消息,RECEIVE用于接收消息。 - -2. msg.sender: 指定消息的发送者,需使用ProjectName@WFName@JobName的格式进行定义。 - -3. msg.topic: 指定消息的主题,建议使用如下格式: 一级分类编码+“”+二级分类编码+“”+三级分类编码。 - -4. msg.name: 指定消息名称,由用户自定义。 - -5. msg.body: 指定消息的内容,没有内容发送可以为空。 - -6. **注意:msg.type默认不可变为SEND,msg.sender、msg.topic、msg.name是必填。** -``` -![signal_01](/images/zh_CN/charpter3/manual/signal_01.png) - - -示例: -``` -msg.type=SEND - -msg.sender=project01@flow@job01 - -msg.topic=bdp_tac_test - -msg.name=TestDynamicReceive - -msg.body=${msg.mycontent} -``` - -### 4.2 eventReceiver节点: - -       EventReceiver节点用于接收eventSender发送过来的消息,并将接收过来的消息内容存放到工作流的上下文中,后续的节点会根据前缀去找该信息进行使用比如作为自定义变量进行使用,eventReceiver支持如下参数: -``` -1. msg.type: 用来指定Job的类型,SEND用于发送消息,RECEIVE用于接收消息。 - -2. msg.receiver: 指定消息的接收者,需使用projectname@jobname@rtxname的格式进行定义。 - -3. msg.topic: 指定消息的主题,建议使用如下格式: 一级分类编码+“”+二级分类编码+“”+三级分类编码。 - -4. msg.name: 指定消息名称,由用户自定义。 - -5. query.frequency: 由于接收消息使用的是主动轮询的方式,wait.time期间的查询次数,。 - -6. max.receive.hours: 最长的接收时长,以小时为单位,超过时长未接收到消息返回失败,。 - -7. msg.savekey: 用于保存消息内容key值,单个flow内多个接收job,需指定不同的msg.savekey保存消息内容,默认值为msg.body,后续Job可以使用该key值获取消息内容。 - -8. msg.rece.today: 如果为true 有且只能接收job启动当天发来的消息 - -9. **注意:msg.type默认不可变为RECEIVE,msg.receiver、msg.topic、msg.name是必填。** -``` -示例使用4.1节的eventSender的信息: -1. 配置reveive接收对应的topic信息,并通过msg.savekey进行保存 -![signal02](/images/zh_CN/charpter3/manual/signal02.png) - -2. 在hql节点中使用receiver的msg.savekey作为自定义变量 -![signal03](/images/zh_CN/charpter3/manual/signal03.png) - -3. 通过上图的运行可以知道整个流程下来hql节点读取到了eventsender发送的信息。信号节点支持跨工程和工作流,这里只是示例使用 - - -### 4.3 dataCheck节点: -       DataCheck节点用于检测数据是否ready,可以判断hive库中某个表或者分区是否存在,如果存在则进行下游的执行,在有数据依赖的任务中扮演十分重要的作用替换掉以前口头约定好的时间开始运行。 -dataCheck支持如下参数: -``` -1. source.type: 依赖的数据来源,job表示是由其他job产生 - -2. data.object: 依赖数据的名称例如:data.object.1=dbname.tablename{partitionlist} - -3. max.check.hours: 描述任务的等待时间,单位是小时 - -4. job.desc: 追加多源信息配置。 -``` -![signal04](/images/zh_CN/charpter3/manual/signal04.png) - - -## 5.数据输出节点 -### 5.1 SendEmail节点 -       SendEmail节点一般作为工作流的最后一个节点,用于将工作流前面的结果信息进行发送,支持发送表格、文本、DashBoard、Display、图片等,用户在使用的时候直接选择发送的对应节点就行: -sendEmail支持如下参数: -``` -类型:支持节点、文字、文件、链接 -邮件标题:指定邮件表提 -发送项:发送的具体内容,例如:类型是节点则这里选择节点 -关联审批单:该邮件是否走过审批,如果未则不会进行发送 -其他邮件基本属性:收件人、抄送、秘密抄送 -``` -![sendemail01](/images/zh_CN/charpter3/manual/sendemail01.png) - -## 6.功能节点 -### 6.1 连接节点: -       Connector节点的作用是为了作为节点与节点的连接,让工作流更加好看: -![connector](/images/zh_CN/charpter3/manual/connector.png) - -### 6.2 子工作流: -       Subflow节点是您可以在一条工作流中嵌入一条子工作流,子工作流支持发布调度,但是在实时执行时父工作流的subflow节点会跳过执行,需要跳到子工作流编辑页面进行执行: -![subflow01](/images/zh_CN/charpter3/manual/subflow01.png) - - - - diff --git a/dss-appconn/appconns/dss-datachecker-appconn/pom.xml b/dss-appconn/appconns/dss-datachecker-appconn/pom.xml new file mode 100644 index 000000000..817fcb72a --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/pom.xml @@ -0,0 +1,158 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-datachecker-appconn + + + + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + + + linkis-common + org.apache.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-development-process-standard-execution + ${dss.version} + + + + log4j + log4j + 1.2.17 + + + + com.squareup.okhttp3 + okhttp + 4.2.2 + + + + org.apache.commons + commons-lang3 + 3.4 + + + + com.alibaba + druid + 1.0.28 + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.properties + **/application.yml + **/bootstrap.yml + **/log4j2.xml + + + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/assembly/distribution.xml b/dss-appconn/appconns/dss-datachecker-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..d71ef2eff --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,84 @@ + + + + dss-datachecker-appconn + + dir + + true + datachecker + + + + + + lib + true + true + false + true + true + + + + + + ${basedir}/src/main/resources + + appconn.properties + + 0777 + / + unix + + + + ${basedir}/src/main/resources + + log4j.properties + log4j2.xml + + 0777 + conf + unix + + + + ${basedir}/src/main/icons + + * + + 0777 + icons + + + + ${basedir}/src/main/resources + + init.sql + + 0777 + db + + + + + + diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/icons/datachecker.icon b/dss-appconn/appconns/dss-datachecker-appconn/src/main/icons/datachecker.icon new file mode 100644 index 000000000..919c7df38 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/icons/datachecker.icon @@ -0,0 +1 @@ +datacheckerCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java new file mode 100644 index 000000000..c1805528b --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker; + + +import com.webank.wedatasphere.dss.appconn.datachecker.connector.DataCheckerDao; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +public class DataChecker { + public final static String SOURCE_TYPE = "source.type"; + public final static String DATA_OBJECT = "check.object"; + public final static String WAIT_TIME = "max.check.hours"; + public final static String QUERY_FREQUENCY = "query.frequency"; + public final static String TIME_SCAPE = "time.scape"; + public final static String MASK_URL = "bdp.mask.url"; + public final static String MASK_APP_ID = "bdp.mask.app.id"; + public final static String MASK_APP_TOKEN = "bdp.mask.app.token"; + + private Properties p; + private static final Logger logger = LoggerFactory.getLogger(DataChecker.class);; + DataCheckerDao wbDao = DataCheckerDao.getInstance(); + DataCheckerExecutionAction dataCheckerAction = null; + public long maxWaitTime; + public int queryFrequency; + + public DataChecker(Properties p, DataCheckerExecutionAction action) { + this.p = p; + dataCheckerAction = action; + maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000; + //test over time +// maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 120 * 1000; + queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "30000")); + + } + + public void run() { + dataCheckerAction.setState(RefExecutionState.Running); + try { + if(p == null) { + throw new RuntimeException("Properties is null. Can't continue"); + } + if (!p.containsKey(SOURCE_TYPE)) { + logger.info("Properties " + SOURCE_TYPE + " value is Null !"); + } + if (!p.containsKey(DATA_OBJECT)) { + logger.info("Properties " + DATA_OBJECT + " value is Null !"); + } + begineCheck(dataCheckerAction); + }catch (Exception ex){ + dataCheckerAction.setState(RefExecutionState.Failed); + throw new RuntimeException("get DataChecker result failed", ex); + } + + } + + public void begineCheck(RefExecutionAction action){ + boolean success=false; + try { + success= wbDao.validateTableStatusFunction(p, logger,action); + }catch (Exception ex){ + dataCheckerAction.setState(RefExecutionState.Failed); + logger.error("datacheck error",ex); + throw new RuntimeException("get DataChecker result failed", ex); + } + if(success) { + dataCheckerAction.setState(RefExecutionState.Success); + }else { + dataCheckerAction.setState(RefExecutionState.Running); + } + } + + public void cancel() { + } + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerAppConn.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerAppConn.java new file mode 100644 index 000000000..19e873518 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerAppConn.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker; + +import com.webank.wedatasphere.dss.appconn.datachecker.standard.DataCheckerDevelopmentStandard; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.appconn.core.impl.AbstractAppConn; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; + +public class DataCheckerAppConn extends AbstractAppConn implements OnlyDevelopmentAppConn { + + private DataCheckerDevelopmentStandard standard; + + @Override + protected void initialize() { + standard = new DataCheckerDevelopmentStandard(); + } + + @Override + public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() { + return standard; + } +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/MaskCheckNotExistException.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/MaskCheckNotExistException.java new file mode 100644 index 000000000..df7fccf73 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/MaskCheckNotExistException.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker.common; + +public class MaskCheckNotExistException extends Exception { + + public MaskCheckNotExistException(final String message) { + super(message); + } +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java new file mode 100644 index 000000000..8cd9a35b7 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java @@ -0,0 +1,404 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker.connector; + +import com.alibaba.druid.pool.DruidDataSource; + +import com.webank.wedatasphere.dss.appconn.datachecker.DataChecker; +import com.webank.wedatasphere.dss.appconn.datachecker.common.MaskCheckNotExistException; +import com.webank.wedatasphere.dss.appconn.datachecker.utils.HttpUtils; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; +import okhttp3.FormBody; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; + +import javax.sql.DataSource; +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class DataCheckerDao { + + private static final String SQL_SOURCE_TYPE_JOB_TABLE = + "SELECT * FROM DBS d JOIN TBLS t ON t.DB_ID = d.DB_ID WHERE d.NAME=? AND t.TBL_NAME=?"; + + private static final String SQL_SOURCE_TYPE_JOB_PARTITION = + "SELECT * FROM DBS d JOIN TBLS t ON t.DB_ID = d.DB_ID JOIN PARTITIONS p ON p.TBL_ID = t.TBL_ID WHERE d.NAME=? AND t.TBL_NAME=? AND p.PART_NAME=?"; + + private static final String SQL_SOURCE_TYPE_BDP = + "SELECT * FROM desktop_bdapimport WHERE bdap_db_name = ? AND bdap_table_name = ? AND target_partition_name = ? AND status = '1';"; + + private static final String SQL_SOURCE_TYPE_BDP_WITH_TIME_CONDITION = + "SELECT * FROM desktop_bdapimport WHERE bdap_db_name = ? AND bdap_table_name = ? AND target_partition_name = ? " + + "AND (UNIX_TIMESTAMP() - UNIX_TIMESTAMP(STR_TO_DATE(modify_time, '%Y-%m-%d %H:%i:%s'))) <= ? AND status = '1';"; + private static final String HIVE_SOURCE_TYPE = "hivedb"; + private static final String MASK_SOURCE_TYPE = "maskdb"; + + private static DataSource jobDS; + private static DataSource bdpDS; + private static DataCheckerDao instance; + + public static DataCheckerDao getInstance() { + if (instance == null) { + synchronized (DataCheckerDao.class) { + if (instance == null) { + instance = new DataCheckerDao(); + } + } + } + return instance; + } + + public boolean validateTableStatusFunction(Properties props, Logger log, RefExecutionAction action) { + if (jobDS == null) { + jobDS = DataDruidFactory.getJobInstance(props, log); + if (jobDS == null) { + log.error("Error getting Druid DataSource instance"); + return false; + } + } + if (bdpDS == null) { + bdpDS = DataDruidFactory.getBDPInstance(props, log); + if (bdpDS == null) { + log.warn("Error getting Druid DataSource instance"); + return false; + } + } + removeBlankSpace(props); + log.info("=============================Data Check Start=========================================="); + + String dataCheckerInfo = props.getProperty(DataChecker.DATA_OBJECT); + if (null != action.getExecutionRequestRefContext()) { + action.getExecutionRequestRefContext().appendLog("=============================Data Check Start=========================================="); +// action.getExecutionRequestRefContext().appendLog("Database table partition info : " + dataCheckerInfo); + } + log.info("(DataChecker info) database table partition info : " + dataCheckerInfo); + long waitTime = Long.valueOf(props.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000; + int queryFrequency = Integer.valueOf(props.getProperty(DataChecker.QUERY_FREQUENCY, "30000")); +// String timeScape = props.getProperty(DataChecker.TIME_SCAPE, "NULL"); + log.info("(DataChecker info) wait time : " + waitTime); + log.info("(DataChecker info) query frequency : " + queryFrequency); +// log.info("(DataChecker info) time scape : " + timeScape); + List> dataObjectList = extractProperties(props); + log.info("DataObjectList size is " + dataObjectList.size()); + dataObjectList.forEach(checkObject -> { + log.info(checkObject.keySet().toString()); + }); + + try (Connection jobConn = jobDS.getConnection(); + Connection bdpConn = bdpDS.getConnection()) { + List allCheckRes = dataObjectList + .stream() + .map(proObjectMap -> { + log.info("Begin to Check dataObject:" + proObjectMap.entrySet().toString()); + boolean checkRes = getDataCheckResult(proObjectMap, jobConn, bdpConn, props, log); + if (null != action.getExecutionRequestRefContext()) { + if (checkRes) { + action.getExecutionRequestRefContext().appendLog("Database table partition info : " + proObjectMap.get(DataChecker.DATA_OBJECT) + " has arrived"); + log.info("sourceType:" + proObjectMap.get(DataChecker.SOURCE_TYPE) + ", Database table partition info : " + proObjectMap.get(DataChecker.DATA_OBJECT) + " has arrived"); + } else { + action.getExecutionRequestRefContext().appendLog("sourceType:" + proObjectMap.get(DataChecker.SOURCE_TYPE) + ", Database table partition info : " + proObjectMap.get(DataChecker.DATA_OBJECT) + " not arrived"); + log.info("sourceType:" + proObjectMap.get(DataChecker.SOURCE_TYPE) + ", Database table partition info : " + proObjectMap.get(DataChecker.DATA_OBJECT) + " not arrived"); + } + } + return checkRes; + }).collect(Collectors.toList()); + boolean flag = allCheckRes.stream().allMatch(res -> res.equals(true)); + if (flag) { + log.info("=============================Data Check End=========================================="); + if (null != action.getExecutionRequestRefContext()) { + action.getExecutionRequestRefContext().appendLog("=============================Data Check End=========================================="); + } + return true; + } + + } catch (SQLException e) { + throw new RuntimeException("get DataChecker result failed", e); + } + + log.info("=============================Data Check End=========================================="); + if (null != action.getExecutionRequestRefContext()) { + action.getExecutionRequestRefContext().appendLog("=============================Data Check End=========================================="); + } + return false; + } + + private boolean getDataCheckResult(Map proObjectMap, Connection jobConn, Connection bdpConn, Properties props, Logger log) { + Predicate> hasDataSource = p -> { + if (StringUtils.isEmpty(proObjectMap.get(DataChecker.SOURCE_TYPE))) { + return false; + } else { + return true; + } + }; + Predicate> hasNotDataSource = hasDataSource.negate(); + Supplier sourceType = () -> proObjectMap.get(DataChecker.SOURCE_TYPE).toLowerCase(); + Predicate> isJobDataSource = p -> sourceType.get().equals("hivedb") || sourceType.get().equals("job"); + Predicate> isBdpDataSource = p -> sourceType.get().equals("maskdb") || sourceType.get().equals("bdp"); + Predicate> isOdsDB = p -> { + String dataObject = proObjectMap.get(DataChecker.DATA_OBJECT) + .replace(" ", "").trim(); + String dbName = dataObject.split("\\.")[0]; + return dbName.contains("_ods"); + }; + Predicate> isNotOdsDB = isOdsDB.negate(); + Predicate> isCheckMetadata = (hasDataSource.and(isJobDataSource)).or(hasNotDataSource.and(isNotOdsDB)); + Predicate> isCheckMask = (hasDataSource.and(isBdpDataSource)).or(hasNotDataSource.and(isOdsDB)); + if (isCheckMetadata.test(proObjectMap)) { + proObjectMap.put(DataChecker.SOURCE_TYPE, HIVE_SOURCE_TYPE); + return getJobTotalCount(proObjectMap, jobConn, log) > 0; + } else { + if (isCheckMask.test(proObjectMap)) { + proObjectMap.put(DataChecker.SOURCE_TYPE, MASK_SOURCE_TYPE); + return (getBdpTotalCount(proObjectMap, bdpConn, log, props) > 0 || "success".equals(fetchMaskCode(proObjectMap, log, props).get("maskStatus"))); + } + return false; + } + + } + + private void sleep(long sleepTime) { + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + private void removeBlankSpace(Properties props) { + try { + props.entrySet().forEach(entry -> { + String value = entry.getValue().toString().replaceAll(" ", "").trim(); + entry.setValue(value); + }); + } catch (Exception e) { + throw new RuntimeException("remove job space char failed", e); + } + } + + private List> extractProperties(Properties p) { + return p.keySet().stream() + .map(key -> key2Map(key, p)).filter(x -> x.size() > 0) + .collect(Collectors.toList()); + } + + private Map key2Map(Object key, Properties p) { + Map proMap = new HashMap<>(); + String skey = String.valueOf(key); + if (skey.contains(DataChecker.DATA_OBJECT)) { + String[] keyArr = skey.split("\\."); + if (keyArr.length == 3) { + String keyNum = keyArr[2]; + String stKey = DataChecker.SOURCE_TYPE + "." + keyNum; + String doKey = DataChecker.DATA_OBJECT + "." + keyNum; + if (null != p.get(stKey)) { + proMap.put(DataChecker.SOURCE_TYPE, String.valueOf(p.get(stKey))); + } + proMap.put(DataChecker.DATA_OBJECT, String.valueOf(p.get(doKey))); + } else { + String stKey = DataChecker.SOURCE_TYPE; + String doKey = DataChecker.DATA_OBJECT; + if (null != p.get(stKey)) { + proMap.put(DataChecker.SOURCE_TYPE, String.valueOf(p.get(stKey))); + } + proMap.put(DataChecker.DATA_OBJECT, String.valueOf(p.get(doKey))); + } + } + + return proMap; + } + + private PreparedStatement getJobStatement(Connection conn, String dataObject) throws SQLException { + String dataScape = dataObject.contains("{") ? "Partition" : "Table"; + String[] dataObjectArray = dataObject.split("\\."); + String dbName = dataObject.split("\\.")[0]; + String tableName = dataObject.split("\\.")[1]; + if (dataScape.equals("Partition")) { + Pattern pattern = Pattern.compile("\\{([^\\}]+)\\}"); + Matcher matcher = pattern.matcher(dataObject); + String partitionName = null; + if (matcher.find()) { + partitionName = matcher.group(1); + } + partitionName = partitionName.replace("\'", "").replace("\"", ""); + tableName = tableName.split("\\{")[0]; + PreparedStatement pstmt = conn.prepareCall(SQL_SOURCE_TYPE_JOB_PARTITION); + pstmt.setString(1, dbName); + pstmt.setString(2, tableName); + pstmt.setString(3, partitionName); + return pstmt; + } else if (dataObjectArray.length == 2) { + PreparedStatement pstmt = conn.prepareCall(SQL_SOURCE_TYPE_JOB_TABLE); + pstmt.setString(1, dbName); + pstmt.setString(2, tableName); + return pstmt; + } else { + throw new SQLException("Error for DataObject format!"); + } + } + + private PreparedStatement getBdpStatement(Connection conn, String dataObject, String timeScape) throws SQLException { + String dataScape = dataObject.contains("{") ? "Partition" : "Table"; + String dbName = dataObject.split("\\.")[0]; + String tableName = dataObject.split("\\.")[1]; + String partitionName = ""; + Pattern pattern = Pattern.compile("\\{([^\\}]+)\\}"); + if (dataScape.equals("Partition")) { + Matcher matcher = pattern.matcher(dataObject); + if (matcher.find()) { + partitionName = matcher.group(1); + } + partitionName = partitionName.replace("\'", "").replace("\"", ""); + tableName = tableName.split("\\{")[0]; + } + PreparedStatement pstmt = null; + if (timeScape.equals("NULL")) { + pstmt = conn.prepareCall(SQL_SOURCE_TYPE_BDP); + } else { + pstmt = conn.prepareCall(SQL_SOURCE_TYPE_BDP_WITH_TIME_CONDITION); + pstmt.setInt(4, Integer.valueOf(timeScape) * 3600); + } + pstmt.setString(1, dbName); + pstmt.setString(2, tableName); + pstmt.setString(3, partitionName); + return pstmt; + } + + private long getJobTotalCount(Map proObjectMap, Connection conn, Logger log) { + String dataObject = proObjectMap.get(DataChecker.DATA_OBJECT); + if (dataObject != null) { + dataObject = dataObject.replace(" ", "").trim(); + } + log.info("-------------------------------------- search hive/spark/mr data "); + log.info("-------------------------------------- : " + dataObject); + try (PreparedStatement pstmt = getJobStatement(conn, dataObject)) { + ResultSet rs = pstmt.executeQuery(); + return rs.last() ? rs.getRow() : 0; + } catch (SQLException e) { + log.error("fetch data from Hive MetaStore error", e); + return 0; + } + } + + private long getBdpTotalCount(Map proObjectMap, Connection conn, Logger log, Properties props) { + String dataObject = proObjectMap.get(DataChecker.DATA_OBJECT); + if (dataObject != null) { + dataObject = dataObject.replace(" ", "").trim(); + } + String timeScape = props.getOrDefault(DataChecker.TIME_SCAPE, "NULL").toString(); + log.info("-------------------------------------- search bdp data "); + log.info("-------------------------------------- : " + dataObject); + try (PreparedStatement pstmt = getBdpStatement(conn, dataObject, timeScape)) { + ResultSet rs = pstmt.executeQuery(); + return rs.last() ? rs.getRow() : 0; + } catch (SQLException e) { + log.error("fetch data from Hive MetaStore error", e); + return 0; + } + } + + private Map fetchMaskCode(Map proObjectMap, Logger log, Properties props) { + log.info("=============================调用BDP MASK接口查询数据状态=========================================="); + Map resultMap = new HashMap(); + String maskUrl = props.getProperty(DataChecker.MASK_URL); + String dataObject = proObjectMap.get(DataChecker.DATA_OBJECT); + if (dataObject != null) { + dataObject = dataObject.replace(" ", "").trim(); + } + String dataScape = dataObject.contains("{") ? "Partition" : "Table"; + String dbName = dataObject.split("\\.")[0]; + String tableName = dataObject.split("\\.")[1]; + String partitionName = ""; + Pattern pattern = Pattern.compile("\\{([^\\}]+)\\}"); + if (dataScape.equals("Partition")) { + Matcher matcher = pattern.matcher(dataObject); + if (matcher.find()) { + partitionName = matcher.group(1); + } + partitionName = partitionName.replace("\'", "").replace("\"", ""); + tableName = tableName.split("\\{")[0]; + } + try { + RequestBody requestBody = new FormBody.Builder() + .add("targetDb", dbName) + .add("targetTable", tableName) + .add("partition", partitionName) + .build(); + Map dataMap = HttpUtils.initSelectParams(props); + log.info("request body:dbName--" + dbName + " tableName--" + tableName + " partitionName--" + partitionName); + Response response = HttpUtils.httpClientHandleBase(maskUrl, requestBody, dataMap); + handleResponse(response, resultMap, log); + } catch (IOException e) { + log.error("fetch data from BDP MASK failed "); + resultMap.put("maskStatus", "noPrepare"); + } catch (MaskCheckNotExistException e) { + String errorMessage = "fetch data from BDP MASK failed" + + "please check database: " + dbName + ",table: " + tableName + "is exist"; + log.error(errorMessage); + throw new RuntimeException(errorMessage, e); + } + return resultMap; + } + + private void handleResponse(Response response, Map proObjectMap, Logger log) + throws IOException, MaskCheckNotExistException { + int responseCode = response.code(); + ResponseBody body = response.body(); + if (responseCode == 200) { + handleResponseBody(body, proObjectMap, log); + } else { + proObjectMap.put("maskStatus", "noPrepare"); + } + } + + private void handleResponseBody(ResponseBody body, Map proObjectMap, Logger log) + throws IOException, MaskCheckNotExistException { + String bodyStr = body.string(); + log.info("mask interface response body:" + bodyStr); + Map entityMap = HttpUtils.getReturnMap(bodyStr); + String codeValue = (String) entityMap.get("code"); + if ("200".equals(codeValue)) { + proObjectMap.put("maskStatus", "success"); + } else if ("1011".equals(codeValue)) { + throw new MaskCheckNotExistException("Mask check failed"); + } else { + proObjectMap.put("maskStatus", "noPrepare"); + } + } + + public static void closeDruidDataSource() { + DruidDataSource jobDSObject = (DruidDataSource) jobDS; + if (jobDSObject != null) { + jobDSObject.close(); + } + } + +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java new file mode 100644 index 000000000..86a2a40c4 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java @@ -0,0 +1,157 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker.connector; + +import com.alibaba.druid.pool.DruidDataSource; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; + +import java.util.Base64; +import java.util.Properties; + +public class DataDruidFactory { + private static DruidDataSource jobInstance; + private static DruidDataSource bdpInstance; + private static DruidDataSource msgInstance; + + public static DruidDataSource getJobInstance(Properties props, Logger log) { + if (jobInstance == null ) { + synchronized (DataDruidFactory.class) { + if(jobInstance == null) { + try { + jobInstance = createDataSource(props, log, "Job"); + } catch (Exception e) { + throw new RuntimeException("Error creating Druid DataSource", e); + } + } + } + } + return jobInstance; + } + public static DruidDataSource getBDPInstance(Properties props, Logger log) { + if (bdpInstance == null ) { + synchronized (DataDruidFactory.class) { + if(bdpInstance == null) { + try { + bdpInstance = createDataSource(props, log, "BDP"); + } catch (Exception e) { + throw new RuntimeException("Error creating Druid DataSource", e); + } + } + } + } + return bdpInstance; + } + + public static DruidDataSource getMsgInstance(Properties props, Logger log) { + if (msgInstance == null ) { + synchronized (DataDruidFactory.class) { + if(msgInstance == null) { + try { + msgInstance = createDataSource(props, log, "Msg"); + } catch (Exception e) { + throw new RuntimeException("Error creating Druid DataSource", e); + } + } + } + } + return msgInstance; + } + + private static DruidDataSource createDataSource(Properties props, Logger log, String type) { + String name = null; + String url = null; + String username = null; + String password = null; + String loginType = null; + if (type.equals("Job")) { + name = props.getProperty("job.datachecker.jdo.option.name"); + url = props.getProperty("job.datachecker.jdo.option.url"); + username = props.getProperty("job.datachecker.jdo.option.username"); + password = props.getProperty("job.datachecker.jdo.option.password"); + loginType = props.getProperty("job.datachecker.jdo.option.login.type"); + log.info("job url is:"+url+"and name is:"+username); + try { + if("base64".equals(loginType)) { + password = new String(Base64.getDecoder().decode(props.getProperty("job.datachecker.jdo.option.password").getBytes()), "UTF-8"); + }else { + password = props.getProperty("job.datachecker.jdo.option.password"); + } + } catch (Exception e){ + log.error("password decore failed" + e); + } + }else if (type.equals("BDP")) { + name = props.getProperty("bdp.datachecker.jdo.option.name"); + url = props.getProperty("bdp.datachecker.jdo.option.url"); + username = props.getProperty("bdp.datachecker.jdo.option.username"); + password = props.getProperty("bdp.datachecker.jdo.option.password"); + loginType = props.getProperty("bdp.datachecker.jdo.option.login.type"); + log.info("bdp url is:"+url+"and name is:"+username); + try { + if("base64".equals(loginType)) { + password = new String(Base64.getDecoder().decode(props.getProperty("bdp.datachecker.jdo.option.password").getBytes()), "UTF-8"); + }else { + password = props.getProperty("bdp.datachecker.jdo.option.password"); + } + } catch (Exception e){ + log.error("password decore failed" + e); + } + } + int initialSize = Integer.valueOf(props.getProperty("datachecker.jdo.option.initial.size", "1")); + int maxActive = Integer.valueOf(props.getProperty("datachecker.jdo.option.max.active", "100")); + int minIdle = Integer.valueOf(props.getProperty("datachecker.jdo.option.min.idle", "1")); + long maxWait = Long.valueOf(props.getProperty("datachecker.jdo.option.max.wait", "60000")); + String validationQuery = props.getProperty("datachecker.jdo.option.validation.quert", "SELECT 'x'"); + long timeBetweenEvictionRunsMillis = Long.valueOf(props.getProperty("datachecker.jdo.option.time.between.eviction.runs.millis", "6000")); + long minEvictableIdleTimeMillis = Long.valueOf(props.getProperty("datachecker.jdo.option.evictable.idle,time.millis", "300000")); + boolean testOnBorrow = Boolean.valueOf(props.getProperty("datachecker.jdo.option.test.on.borrow", "true")); + int maxOpenPreparedStatements = Integer.valueOf(props.getProperty("datachecker.jdo.option.max.open.prepared.statements", "-1")); + + + if (timeBetweenEvictionRunsMillis > minEvictableIdleTimeMillis) { + timeBetweenEvictionRunsMillis = minEvictableIdleTimeMillis; + } + + DruidDataSource ds = new DruidDataSource(); + + if (StringUtils.isNotBlank(name)) { + ds.setName(name); + } + + ds.setUrl(url); + ds.setDriverClassName("com.mysql.jdbc.Driver"); + ds.setUsername(username); + ds.setPassword(password); + ds.setInitialSize(initialSize); + ds.setMinIdle(minIdle); + ds.setMaxActive(maxActive); + ds.setMaxWait(maxWait); + ds.setTestOnBorrow(testOnBorrow); + ds.setValidationQuery(validationQuery); + ds.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); + ds.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); + if (maxOpenPreparedStatements > 0) { + ds.setPoolPreparedStatements(true); + ds.setMaxPoolPreparedStatementPerConnectionSize( + maxOpenPreparedStatements); + } else { + ds.setPoolPreparedStatements(false); + } + log.info("Druid data source initialed!"); + return ds; + } +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/service/DataCheckerExecuteService.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/service/DataCheckerExecuteService.java new file mode 100644 index 000000000..de302e66e --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/service/DataCheckerExecuteService.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker.service; + +import com.webank.wedatasphere.dss.appconn.datachecker.DataCheckerRefExecutionOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExecutionService; + +public class DataCheckerExecuteService extends AbstractRefExecutionService { + + @Override + protected DataCheckerRefExecutionOperation createRefExecutionOperation() { + return new DataCheckerRefExecutionOperation(); + } + +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/standard/DataCheckerDevelopmentStandard.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/standard/DataCheckerDevelopmentStandard.java new file mode 100644 index 000000000..44803691a --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/standard/DataCheckerDevelopmentStandard.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker.standard; + +import com.webank.wedatasphere.dss.appconn.datachecker.service.DataCheckerExecuteService; +import com.webank.wedatasphere.dss.standard.app.development.standard.OnlyExecutionDevelopmentStandard; + +public class DataCheckerDevelopmentStandard extends OnlyExecutionDevelopmentStandard { + + @Override + protected DataCheckerExecuteService createRefExecutionService() { + return new DataCheckerExecuteService(); + } + + @Override + public void init() { + } + + +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/utils/HttpUtils.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/utils/HttpUtils.java new file mode 100644 index 000000000..85a9284f1 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/utils/HttpUtils.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker.utils; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import com.webank.wedatasphere.dss.appconn.datachecker.DataChecker; +import okhttp3.*; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class HttpUtils { + + private static final Logger logger = Logger.getLogger(HttpUtils.class); + + public static Response httpClientHandleBase(String actionUrl, RequestBody requestBody, Map urlMap) throws IOException { + String maskUrl = actionUrl + "appid=" + urlMap.get("appid") + "&&nonce=" + urlMap.get("nonce") + + "&×tamp=" + urlMap.get("timestamp") + "&&signature=" + urlMap.get("signature"); + OkHttpClient okHttpClient = new OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .writeTimeout(20, TimeUnit.SECONDS) + .readTimeout(20, TimeUnit.SECONDS) + .build(); + + logger.info("access mask URL is:"+maskUrl); + Request request = new Request.Builder() + .url(maskUrl) + .post(requestBody) + .build(); + Call call = okHttpClient.newCall(request); + Response response = call.execute(); + logger.info("mask interface response code:" + response.code()); + return response; + } + + public static String httpClientHandle(String actionUrl, RequestBody requestBody, Map urlMap) throws Exception{ + String returnData = ""; + try { + Response response = httpClientHandleBase(actionUrl, requestBody, urlMap); + returnData = response.body().string(); + logger.info("mask interface return message:" + returnData); + } catch (IOException e) { + e.printStackTrace(); + } + return returnData; + } + + public static String httpClientHandle(String actionUrl) throws Exception{ + OkHttpClient okHttpClient = new OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .writeTimeout(20, TimeUnit.SECONDS) + .readTimeout(20, TimeUnit.SECONDS) + .build(); + Request request = new Request.Builder() + .url(actionUrl) + .build(); + Call call = okHttpClient.newCall(request); + String returnData = ""; + try { + Response response = call.execute(); + returnData = response.body().string(); + logger.info("interface return message:" + returnData); + } catch (IOException e) { + e.printStackTrace(); + } + return returnData; + } + + public static Map getReturnMap(String dataStr){ + Map dataMap = new HashMap<>(); + GsonBuilder gb = new GsonBuilder(); + Gson g = gb.create(); + dataMap = g.fromJson(dataStr, new TypeToken>(){}.getType()); + return dataMap; + } + + public static Map initSelectParams(Properties props){ + String appid = props.getProperty(DataChecker.MASK_APP_ID); + String token = props.getProperty(DataChecker.MASK_APP_TOKEN); + String nonce = RandomStringUtils.random(5, "0123456789"); + Long cur_time = System.currentTimeMillis() / 1000; + Map requestProperties = new HashMap<>(); + requestProperties.put("appid", appid); + requestProperties.put("nonce", nonce.toString()); + requestProperties.put("signature", getMD5(getMD5(appid + nonce.toString() + cur_time) + token)); + requestProperties.put("timestamp", cur_time.toString()); + return requestProperties; + } + + public static String getMD5(String str){ + return DigestUtils.md5Hex(str.getBytes()); + } + +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/appconn.properties b/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/appconn.properties new file mode 100644 index 000000000..ac52d9b55 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/appconn.properties @@ -0,0 +1,33 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +job.datachecker.jdo.option.name=job +job.datachecker.jdo.option.url=jdbc:mysql://127.0.0.1:3306/ +job.datachecker.jdo.option.username= +job.datachecker.jdo.option.password= +job.datachecker.jdo.option.login.type=base64 + +bdp.datachecker.jdo.option.name=bdp +bdp.datachecker.jdo.option.url= +bdp.datachecker.jdo.option.username= +bdp.datachecker.jdo.option.password= +bdp.datachecker.jdo.option.login.type=base64 + + +bdp.mask.url= +bdp.mask.app.id= +bdp.mask.app.token= + diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/init.sql b/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/init.sql new file mode 100644 index 000000000..69776cfe8 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/init.sql @@ -0,0 +1,47 @@ +-- TODO 这里只适用于第一次安装时。如果是更新的话dss_appconn表不能先删除再插入,因为其他表如dss_workspace_appconn_role关联了appconn_id(不能变),需要使用update、alter语句更新 +select @datachecker_appconnId:=id from `dss_appconn` where `appconn_name` = 'datachecker'; +delete from `dss_appconn_instance` where `appconn_id` = @datachecker_appconnId; + +delete from dss_appconn where appconn_name = "datachecker"; +INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) +VALUES ('datachecker', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.dss.appconn.datachecker.DataCheckerAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/datachecker', ''); + +select @datachecker_appconnId:=id from `dss_appconn` where `appconn_name` = 'datachecker'; + +INSERT INTO `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`) +VALUES (@datachecker_appconnId, 'DEV', 'datachecker', '{\"job.datachecker.jdo.option.name\":\"job\",\"job.datachecker.jdo.option.url\":\"DATACHECKER_JOB_JDBC_URL\",\"job.datachecker.jdo.option.username\":\"DATACHECKER_JOB_JDBC_USERNAME\",\"job.datachecker.jdo.option.password\":\"DATACHECKER_JOB_JDBC_PASSWORD\",\"bdp.datachecker.jdo.option.name\":\"bdp\",\"bdp.datachecker.jdo.option.url\":\"DATACHECKER_BDP_JDBC_URL\",\"bdp.datachecker.jdo.option.username\":\"DATACHECKER_BDP_JDBC_USERNAME\",\"bdp.datachecker.jdo.option.password\":\"DATACHECKER_BDP_JDBC_PASSWORD\",\"bdp.datachecker.jdo.option.login.type\":\"base64\",\"bdp.mask.url\":\"http://BDP_MASK_IP:BDP_MASK_PORT/api/v1/mask-status?\",\"bdp.mask.app.id\":\"wtss\",\"bdp.mask.app.token\":\"20a0ccdfc0\"}', ''); + +delete from dss_workflow_node where appconn_name = "datachecker"; +insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) +values('datachecker','datachecker','linkis.appconn.datachecker','0','0','1','1','0','icons/datachecker.icon'); + +select @datachecker_nodeId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.datachecker'; + +delete from `dss_workflow_node_to_group` where `node_id`=@datachecker_nodeId; +delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@datachecker_nodeId; + +-- 查找节点所属组的id +select @datachecker_node_groupId:=id from `dss_workflow_node_group` where `name` = '信号节点'; + +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@datachecker_nodeId, @datachecker_node_groupId); + +-- 考虑表中有的是重复记录,最好加上limit 1 +select @datachecker_node_ui_lable_name_1:=id from `dss_workflow_node_ui` where `lable_name` = '节点名' limit 1; +select @datachecker_node_ui_lable_name_2:=id from `dss_workflow_node_ui` where `lable_name` = '节点描述' limit 1; +select @datachecker_node_ui_lable_name_3:=id from `dss_workflow_node_ui` where `lable_name` = '业务标签' limit 1; +select @datachecker_node_ui_lable_name_4:=id from `dss_workflow_node_ui` where `lable_name` = '应用标签' limit 1; +select @datachecker_node_ui_lable_name_5:=id from `dss_workflow_node_ui` where `lable_name` = '是否复用引擎' limit 1; +select @datachecker_node_ui_lable_name_6:=id from `dss_workflow_node_ui` where `lable_name` = 'source.type' limit 1; +select @datachecker_node_ui_lable_name_7:=id from `dss_workflow_node_ui` where `lable_name` = 'check.object' limit 1; +select @datachecker_node_ui_lable_name_8:=id from `dss_workflow_node_ui` where `lable_name` = 'max.check.hours' limit 1; +select @datachecker_node_ui_lable_name_9:=id from `dss_workflow_node_ui` where `lable_name` = 'job.desc' limit 1; + +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_4); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_5); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_6); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_7); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_8); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@datachecker_nodeId, @datachecker_node_ui_lable_name_9); diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/log4j.properties b/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/log4j.properties new file mode 100644 index 000000000..ee8619595 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/log4j2.xml b/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8c40a73e8 --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/resources/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerExecutionAction.scala b/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerExecutionAction.scala new file mode 100644 index 000000000..9a76b1bed --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerExecutionAction.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker + +import com.webank.wedatasphere.dss.standard.app.development.listener.common.AbstractRefExecutionAction +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef + +class DataCheckerExecutionAction extends AbstractRefExecutionAction { + + private var response: ExecutionResponseRef = _ + + def setExecutionResponseRef(response: ExecutionResponseRef): Unit = + this.response = response + + def getExecutionResponseRef: ExecutionResponseRef = response + + private[this] var _dc: DataChecker = _ + + def dc: DataChecker = _dc + + def setDc(value: DataChecker): Unit = { + _dc = value + } + +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerRefExecutionOperation.scala b/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerRefExecutionOperation.scala new file mode 100644 index 000000000..f9511f39c --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerRefExecutionOperation.scala @@ -0,0 +1,168 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.datachecker + +import com.webank.wedatasphere.dss.common.utils.VariableUtils + +import java.util +import java.util.{Properties, UUID} +import com.webank.wedatasphere.dss.standard.app.development.listener.common._ +import com.webank.wedatasphere.dss.standard.app.development.listener.core.{Killable, LongTermRefExecutionOperation, Procedure} +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef.ExecutionResponseRefBuilder +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.{AsyncExecutionResponseRef, ExecutionResponseRef, RefExecutionRequestRef} +import org.apache.linkis.common.log.LogUtils +import org.apache.linkis.common.utils.Utils + +import scala.collection.mutable + +class DataCheckerRefExecutionOperation + extends LongTermRefExecutionOperation[RefExecutionRequestRef.RefExecutionRequestRefImpl] with Killable with Procedure{ + + protected def putErrorMsg(errorMsg: String, t: Throwable, action: DataCheckerExecutionAction): DataCheckerExecutionAction = { + val responseRef = new ExecutionResponseRefBuilder().setErrorMsg(errorMsg).setException(t).error() + action.setExecutionResponseRef(responseRef) + action + } + + override def submit(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl): RefExecutionAction = { + val nodeAction = new DataCheckerExecutionAction() + nodeAction.setId(UUID.randomUUID().toString) + import scala.collection.JavaConversions.mapAsScalaMap + val InstanceConfig = this.service.getAppInstance.getConfig + val runTimeParams = requestRef.getExecutionRequestRefContext.getRuntimeMap + val variableParams: mutable.Map[String, Object]= requestRef.getRefJobContent.get("variable"). asInstanceOf[java.util.Map[String,Object]] + val inputParams = runTimeParams ++ variableParams + val properties = new Properties() + InstanceConfig.foreach { + case (key: String, value: Object) => + //避免密码被打印 + properties.put(key, value.toString) + } + val tmpProperties = new Properties() + runTimeParams.foreach( + record=> + if (null == record._2) { + properties.put(record._1, "") + }else { + if (record._1.equalsIgnoreCase("job.desc")) { + val descValue = record._2.asInstanceOf[String] + var rows=Array.empty[String] + if (descValue.contains("\n")) { + rows = descValue.split("\n") + }else{ + rows= descValue.split(";") + } + rows.foreach(row => if (row.contains("=")) { + val endLocation = row.indexOf("=") + val rowKey = row.substring(0, endLocation) + val rowEnd = row.substring(endLocation + 1) + tmpProperties.put(rowKey, rowEnd) + }) + } else { + tmpProperties.put(record._1, record._2) + } + } + ) + tmpProperties.foreach { record => + logger.info("request params key : " + record._1 + ",value : " + record._2) + if (null == record._2) { + properties.put(record._1, "") + } + else { + if (inputParams.exists(x => x._1.equalsIgnoreCase(VariableUtils.RUN_DATE))) { + val tmp: util.HashMap[String, Any] = new util.HashMap[String, Any]() + tmp.put(VariableUtils.RUN_DATE, inputParams.getOrElse(VariableUtils.RUN_DATE, null)) + properties.put(record._1, VariableUtils.replace(record._2.toString, tmp)) + } else { + properties.put(record._1, VariableUtils.replace(record._2.toString)) + } + } + } + Utils.tryCatch({ + val dc = new DataChecker(properties, nodeAction) + dc.run() + nodeAction.setDc(dc) + })(t => { + logger.error("DataChecker run failed for " + t.getMessage, t) + putErrorMsg("DataChecker run failed! " + t.getMessage, t, nodeAction) + }) + nodeAction + + } + + override def state(action: RefExecutionAction): RefExecutionState = { + action match { + case action: DataCheckerExecutionAction => + action.getExecutionRequestRefContext.appendLog("DataCheck is running!") + if (action.getState.isCompleted) return action.getState + Utils.tryCatch(action.dc.begineCheck(action))(t => { + action.setState(RefExecutionState.Failed) + logger.error("DataChecker run failed for " + t.getMessage, t) + putErrorMsg("DataChecker run failed! " + t.getMessage, t, action) + }) + action.getState + case _ => RefExecutionState.Failed + } + } + + override def result(action: RefExecutionAction): ExecutionResponseRef = { + action match { + case action: DataCheckerExecutionAction => + if (action.getState.equals(RefExecutionState.Success)) { + new ExecutionResponseRefBuilder().success() + } else if(action.getExecutionResponseRef != null) action.getExecutionResponseRef + else new ExecutionResponseRefBuilder().error() + case _ => + new ExecutionResponseRefBuilder().error() + } + } + + override def kill(action: RefExecutionAction): Boolean = action match { + case longTermAction: DataCheckerExecutionAction => + longTermAction.setKilledFlag(true) + longTermAction.setState(RefExecutionState.Killed) + true + } + + override def progress(action: RefExecutionAction): Float = { + //todo complete progress + 0.5f + } + + override def log(action: RefExecutionAction): String = { + action match { + case action: DataCheckerExecutionAction => + if (!action.getState.isCompleted) { + LogUtils.generateInfo("DataChecker is waiting for tables") + } else { + LogUtils.generateInfo("DataChecker successfully received info of tables") + } + case _ => LogUtils.generateERROR("Error for NodeExecutionAction ") + } + + } + + override def createAsyncResponseRef(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl, action: RefExecutionAction): AsyncExecutionResponseRef = { + action match { + case action: DataCheckerExecutionAction => + val response = super.createAsyncResponseRef(requestRef, action) + new AsyncExecutionResponseRef.Builder().setMaxLoopTime(action.dc.maxWaitTime) + .setAskStatePeriod(action.dc.queryFrequency).setAsyncExecutionResponseRef(response).build() + } + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml b/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml new file mode 100644 index 000000000..fc00a8473 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml @@ -0,0 +1,139 @@ + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-dolphinscheduler-appconn + + + + com.webank.wedatasphere.dss + dss-scheduler-appconn + ${dss.version} + + provided + + + + com.webank.wedatasphere.dss + dss-origin-sso-integration-standard + ${dss.version} + + + linkis-common + org.apache.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + + org.springframework + spring-beans + 5.2.12.RELEASE + + + com.google.code.gson + gson + ${gson.version} + provided + + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + provided + + + com.webank.wedatasphere.dss + dss-contextservice + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.linkis + linkis-storage-script-dev-server + 1.0.3 + + + com.google.guava + guava + 30.0-jre + + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/assembly/distribution.xml b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..66e5d3067 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,78 @@ + + + + dss-dolphinscheduler-appconn + + dir + + true + dolphinscheduler + + + + + + lib + true + true + false + true + true + + + + + + ${basedir}/src/main/resources + + appconn.properties + + 0777 + / + unix + + + + ${basedir}/src/main/resources + + log4j.properties + log4j2.xml + + 0777 + conf + unix + + + + ${basedir}/src/main/resources + + init.sql + + 0777 + db + + + + + + + diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/DolphinSchedulerAppConn.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/DolphinSchedulerAppConn.java new file mode 100644 index 000000000..54a307bb5 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/DolphinSchedulerAppConn.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler; + +import com.webank.wedatasphere.dss.appconn.core.ext.OptionalAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.standard.DolphinSchedulerStructureStandard; +import com.webank.wedatasphere.dss.appconn.scheduler.AbstractSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerStructureIntegrationStandard; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ConversionIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.structure.OptionalIntegrationStandard; +import com.webank.wedatasphere.dss.workflow.conversion.WorkflowConversionIntegrationStandard; + +public class DolphinSchedulerAppConn extends AbstractSchedulerAppConn implements OptionalAppConn { + + public static final String DOLPHINSCHEDULER_APPCONN_NAME = "dolphinscheduler"; + + @Override + public ConversionIntegrationStandard createConversionIntegrationStandard() { + if (super.getOrCreateConversionStandard() == null) { + return new WorkflowConversionIntegrationStandard(); + } else { + return super.getOrCreateConversionStandard(); + } + } + + @Override + public SchedulerStructureIntegrationStandard getOrCreateStructureStandard() { + return DolphinSchedulerStructureStandard.getInstance(); + } + + @Override + public OptionalIntegrationStandard getOrCreateOptionalStandard() { + return OptionalAppConn.super.getOrCreateOptionalStandard(); + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conf/DolphinSchedulerConf.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conf/DolphinSchedulerConf.java new file mode 100644 index 000000000..4d0f693f7 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conf/DolphinSchedulerConf.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; + +public interface DolphinSchedulerConf { + + CommonVars DS_ADMIN_USER = CommonVars.apply("wds.dss.appconn.ds.admin.user", "admin"); + + CommonVars DS_ADMIN_TOKEN = CommonVars.apply("wds.dss.appconn.ds.admin.token", ""); + + CommonVars DS_TOKEN_EXPIRE_TIME = CommonVars.apply("wds.dss.appconn.ds.token.expire.time", new TimeType("2h")); + + CommonVars DS_TOKEN_EXPIRE_TIME_GAP = CommonVars.apply("wds.dss.appconn.ds.token.expire.gap", new TimeType("1m")); + + CommonVars DS_VERSION = CommonVars.apply("wds.dss.appconn.ds.version", "1.3.9"); + + CommonVars DSS_DOLPHINSCHEDULER_CLIENT_HOME = CommonVars.apply("wds.dss.appconn.ds.client.home", "${DSS_DOLPHINSCHEDULER_CLIENT_HOME}"); + + CommonVars DOLPHIN_SCHEDULER_URI_PREFIX = CommonVars.apply("wds.dss.appconn.ds.url.prefix", "dolphinscheduler"); + + CommonVars LINKIS_1_X_VERSION = CommonVars.apply("wds.dss.appconn.ds.linkis.version", "1.0.0"); + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/constant/Constant.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/constant/Constant.java new file mode 100644 index 000000000..4c331eb33 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/constant/Constant.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.constant; + +public class Constant { + + public final static String LINKIS_TYPE = "linkistype"; + public final static String JOB_COMMAND = "command"; + public final static String PROXY_USER = "proxy.user"; + + /** + * dolphin scheduler result code. + */ + public final static int DS_RESULT_CODE_SUCCESS = 0; + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/DolphinSchedulerWorkflowToRelConverter.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/DolphinSchedulerWorkflowToRelConverter.java new file mode 100644 index 000000000..8b0e11202 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/DolphinSchedulerWorkflowToRelConverter.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.conversion; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerConvertedRel; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.WorkflowPreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.operation.WorkflowToRelConverter; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import org.apache.commons.lang.StringUtils; + +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class DolphinSchedulerWorkflowToRelConverter implements WorkflowToRelConverter { + + @Override + public ConvertedRel convertToRel(PreConversionRel rel) { + Workflow workflow = ((WorkflowPreConversionRel) rel).getWorkflow(); + String repeatNodes = workflow.getWorkflowNodes().stream().map(WorkflowNode::getName) + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())) + .entrySet().stream().filter(entry -> entry.getValue() > 1).map(Map.Entry::getKey) + .collect(Collectors.joining(", ")); + if (StringUtils.isNotEmpty(repeatNodes)) { + throw new DSSRuntimeException(80001, "重复的节点名称:" + repeatNodes); + } + DolphinSchedulerConvertedRel dolphinSchedulerConvertedRel = new DolphinSchedulerConvertedRel(rel); + return dolphinSchedulerConvertedRel; + } + + @Override + public int getOrder() { + return 5; + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/NodeConverter.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/NodeConverter.java new file mode 100644 index 000000000..68cfa4a2c --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/NodeConverter.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.conversion; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerConvertedRel; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerTask; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerTaskParam; +import com.webank.wedatasphere.dss.appconn.scheduler.utils.SchedulerConf; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.OrchestrationToRelConversionRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowWithContextImpl; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.linkis.common.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; + +import static com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf.DSS_DOLPHINSCHEDULER_CLIENT_HOME; + +public class NodeConverter { + + private static final Logger logger = LoggerFactory.getLogger(NodeConverter.class); + + public NodeConverter() { + } + + /** + * 将DSS中节点转为 DolphinScheduler 中节点 task. + * + * @param dssNode + * the scheduler node + * @return the dolphin scheduler task + */ + public DolphinSchedulerTask convertNode(DolphinSchedulerConvertedRel dolphinSchedulerConvertedRel, + DSSNode dssNode) { + OrchestrationToRelConversionRequestRef ref = dolphinSchedulerConvertedRel.getDSSToRelConversionRequestRef(); + DolphinSchedulerTask task = new DolphinSchedulerTask(); + task.setId(dssNode.getId()); + task.setName(dssNode.getName()); + task.setPreTasks(dssNode.getDependencys()); + task.setType("SHELL"); + String nodeStr = String.join("-", ref.getWorkspace().getWorkspaceName(), + ref.getDSSProject().getName(), ref.getDSSOrchestration().getName(), dssNode.getName()); + Map sourceMap = MapUtils.newCommonMapBuilder().put("workspaceName", ref.getWorkspace().getWorkspaceName()) + .put("projectName", ref.getDSSProject().getName()) + .put("flowName", ref.getDSSOrchestration().getName()).put("nodeName", dssNode.getName()).build(); + WorkflowWithContextImpl workflow = (WorkflowWithContextImpl) dolphinSchedulerConvertedRel.getWorkflow(); + DolphinSchedulerTaskParam taskParams = new DolphinSchedulerTaskParam(); + try { + List scriptList = new ArrayList<>(); + BiConsumer addLine = (key, value) -> scriptList.add(String.format("export %s='%s'", key, value)); + BiConsumer addObjectLine = (key, value) -> { + if(value == null) { + return; + } + String valueStr = DSSCommonUtils.COMMON_GSON.toJson(value); + valueStr = valueStr.replaceAll("\"", "\\\""); + addLine.accept(key, valueStr); + }; + addLine.accept("LINKIS_TYPE", dssNode.getNodeType()); + addLine.accept("PROXY_USER", dssNode.getUserProxy()); + addLine.accept("LINKIS_VERSION", DolphinSchedulerConf.LINKIS_1_X_VERSION.getValue()); + addObjectLine.accept("JOB_COMMAND", dssNode.getJobContent()); + addObjectLine.accept("JOB_PARAMS", dssNode.getParams()); + addObjectLine.accept("JOB_RESOURCES", dssNode.getResources()); + addObjectLine.accept("JOB_SOURCE", sourceMap); + addLine.accept("CONTEXT_ID", workflow.getContextID()); + addLine.accept("LINKIS_GATEWAY_URL", Configuration.getGateWayURL()); + addLine.accept("DS_VERSION", DolphinSchedulerConf.DS_VERSION.getValue()); + //todo + addLine.accept("RUN_DATE", "${global_run_date}"); + addObjectLine.accept("JOB_LABELS", new EnvDSSLabel(SchedulerConf.JOB_LABEL.getValue()).getValue()); + if(CollectionUtils.isNotEmpty(workflow.getFlowResources())) { + addObjectLine.accept("FLOW_RESOURCES", workflow.getFlowResources()); + } + if(CollectionUtils.isNotEmpty(workflow.getFlowProperties())) { + addObjectLine.accept("FLOW_PROPERTIES", workflow.getFlowProperties()); + } + String executionScript = String.join(" ", "sh", + DSS_DOLPHINSCHEDULER_CLIENT_HOME.getValue() + "/bin/dss-dolphinscheduler-client.sh", + nodeStr); + scriptList.add(executionScript); + taskParams.setRawScript(String.join("\n", scriptList)); + } catch (Exception e) { + logger.error("工作流节点 {} 转换失败.", dssNode.getName(), e); + throw new ExternalOperationWarnException(90321, "工作流节点 " + dssNode.getName() + " 转换成DolphinScheduler节点失败!", e); + } + task.setParams(taskParams); + return task; + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/WorkflowToDolphinSchedulerRelConverter.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/WorkflowToDolphinSchedulerRelConverter.java new file mode 100644 index 000000000..2a3df0893 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/WorkflowToDolphinSchedulerRelConverter.java @@ -0,0 +1,113 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.conversion; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerConvertedRel; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerTask; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerWorkflow; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.operation.WorkflowToRelConverter; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNodeEdge; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class WorkflowToDolphinSchedulerRelConverter implements WorkflowToRelConverter { + + public static final Logger logger = LoggerFactory.getLogger(WorkflowToDolphinSchedulerRelConverter.class); + + private static NodeConverter nodeConverter = new NodeConverter(); + + @Override + public ConvertedRel convertToRel(PreConversionRel rel) { + DolphinSchedulerConvertedRel dolphinSchedulerConvertedRel = (DolphinSchedulerConvertedRel)rel; + List workflowNodes = dolphinSchedulerConvertedRel.getWorkflow().getWorkflowNodes(); + for (WorkflowNode workflowNode : workflowNodes) { + if ("workflow.subflow".equals(workflowNode.getNodeType())) { + throw new ExternalOperationFailedException(90021, workflowNode.getName() + "当前不支持将subFlow节点发布到DolphinScheduler!"); + } + } + Workflow dolphinSchedulerWorkflow = convertWorkflow(dolphinSchedulerConvertedRel); + dolphinSchedulerConvertedRel.setWorkflow(dolphinSchedulerWorkflow); + return dolphinSchedulerConvertedRel; + } + + @Override + public int getOrder() { + return 10; + } + + private DolphinSchedulerWorkflow convertWorkflow(DolphinSchedulerConvertedRel dolphinSchedulerConvertedRel) { + DolphinSchedulerWorkflow dolphinSchedulerWorkflow = new DolphinSchedulerWorkflow(); + Workflow workflow = dolphinSchedulerConvertedRel.getWorkflow(); + try { + BeanUtils.copyProperties(workflow, dolphinSchedulerWorkflow); + } catch (Exception e) { + throw new DSSRuntimeException(91500, "Copy workflow fields failed!", e); + } + DolphinSchedulerWorkflow.ProcessDefinitionJson processDefinitionJson = + new DolphinSchedulerWorkflow.ProcessDefinitionJson(); + processDefinitionJson.setGlobalParams(convertGlobalParams(workflow.getFlowProperties())); + processDefinitionJson.getGlobalParams().add(new HashMap() {{ + put("prop", "global_run_date"); + put("value", "${system.biz.date}"); + put("type", "VARCHAR"); + put("direct", "IN"); + }}); + Map locations = new HashMap<>(); + Map nameToIdMap = workflow.getWorkflowNodes().stream() + .collect(Collectors.toMap(WorkflowNode::getName, WorkflowNode::getId)); + for (WorkflowNode workflowNode : workflow.getWorkflowNodes()) { + DSSNode node = workflowNode.getDSSNode(); + if (node.getLayout() == null) { + continue; + } + DolphinSchedulerTask dolphinSchedulerTask = nodeConverter.convertNode(dolphinSchedulerConvertedRel, node); + processDefinitionJson.addTask(dolphinSchedulerTask); + + DolphinSchedulerWorkflow.LocationInfo locationInfo = new DolphinSchedulerWorkflow.LocationInfo(); + locationInfo.setName(node.getName()); + String targetArr = node.getDependencys().stream().map(nameToIdMap::get).collect(Collectors.joining(",")); + locationInfo.setTargetarr(targetArr); + locationInfo.setX((int)node.getLayout().getX()); + locationInfo.setY((int)node.getLayout().getY()); + locations.put(node.getId(), locationInfo); + } + + List workflowNodeEdges = workflow.getWorkflowNodeEdges(); + List connects = new LinkedList<>(); + for (WorkflowNodeEdge edge : workflowNodeEdges) { + connects.add( + new DolphinSchedulerWorkflow.Connect(edge.getDSSEdge().getSource(), edge.getDSSEdge().getTarget())); + } + + dolphinSchedulerWorkflow.setProcessDefinitionJson(processDefinitionJson); + dolphinSchedulerWorkflow.setLocations(locations); + dolphinSchedulerWorkflow.setConnects(connects); + + return dolphinSchedulerWorkflow; + } + + private List> convertGlobalParams(List> globalParams) { + return globalParams.stream().map(map -> { + Map ret = new HashMap<>(); + map.forEach((k, v) -> { + ret.put("prop", k); + ret.put("value", v); + ret.put("type", "VARCHAR"); + ret.put("direct", "IN"); + }); + return ret; + }).collect(Collectors.toList()); + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/WorkflowToDolphinSchedulerSynchronizer.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/WorkflowToDolphinSchedulerSynchronizer.java new file mode 100644 index 000000000..ecff05d7f --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/conversion/WorkflowToDolphinSchedulerSynchronizer.java @@ -0,0 +1,60 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.conversion; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerConvertedRel; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationUpdateOperation; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationUpdateRequestRef; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.DSSToRelConversionOperation; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.OrchestrationToRelConversionRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.operation.WorkflowToRelSynchronizer; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import org.apache.commons.collections.CollectionUtils; + +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class WorkflowToDolphinSchedulerSynchronizer implements WorkflowToRelSynchronizer { + + private DSSToRelConversionOperation dssToRelConversionOperation; + + + @Override + public void setDSSToRelConversionOperation(DSSToRelConversionOperation dssToRelConversionOperation) { + this.dssToRelConversionOperation = dssToRelConversionOperation; + } + + @Override + public void syncToRel(ConvertedRel convertedRel) { + DolphinSchedulerConvertedRel dolphinSchedulerConvertedRel = (DolphinSchedulerConvertedRel)convertedRel; + OrchestrationToRelConversionRequestRef requestRef = dolphinSchedulerConvertedRel.getDSSToRelConversionRequestRef(); + Workflow workflow = dolphinSchedulerConvertedRel.getWorkflow(); +// checkSchedulerProject(workflow); + Long dolphinSchedulerWorkflowId = requestRef.getRefOrchestrationId(); + DolphinSchedulerAppConn appConn = (DolphinSchedulerAppConn) dssToRelConversionOperation.getConversionService().getAppStandard().getAppConn(); + OrchestrationUpdateOperation updateOperation = appConn.getOrCreateStructureStandard().getOrchestrationService(dssToRelConversionOperation.getConversionService().getAppInstance()) + .getOrchestrationUpdateOperation(); + OrchestrationUpdateRequestRef.OrchestrationUpdateRequestRefImpl ref = new OrchestrationUpdateRequestRef.OrchestrationUpdateRequestRefImpl() + .setRefOrchestrationId(dolphinSchedulerWorkflowId).setProjectName(requestRef.getDSSProject().getName()) + .setUserName(requestRef.getUserName()).setWorkspace(requestRef.getWorkspace()).setDSSOrchestration(workflow); + updateOperation.updateOrchestration(ref); + } + +// private void checkSchedulerProject(Workflow flow) throws ExternalOperationFailedException { +// List nodes = flow.getWorkflowNodes(); +// for (WorkflowNode node : nodes) { +// DSSNode dssNode = node.getDSSNode(); +// if (CollectionUtils.isEmpty(dssNode.getResources()) && dssNode.getJobContent().isEmpty()) { +// throw new ExternalOperationFailedException(90021, dssNode.getName() + "节点内容不能为空"); +// } +// } +// } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerAccessToken.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerAccessToken.java new file mode 100644 index 000000000..ac018bb8e --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerAccessToken.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity; + +import java.util.Date; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DolphinSchedulerAccessToken { + + private int id; + + private int userId; + + private String userName; + + private String token; + + private Date expireTime; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getUserId() { + return userId; + } + + public void setUserId(int userId) { + this.userId = userId; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } + + public Date getExpireTime() { + return expireTime; + } + + public void setExpireTime(Date expireTime) { + this.expireTime = expireTime; + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerConvertedRel.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerConvertedRel.java new file mode 100644 index 000000000..041491dba --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerConvertedRel.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.OrchestrationToRelConversionRequestRef; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.WorkflowPreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.WorkflowPreConversionRelImpl; + +public class DolphinSchedulerConvertedRel extends WorkflowPreConversionRelImpl implements ConvertedRel { + + public DolphinSchedulerConvertedRel(PreConversionRel rel) { + setWorkflow(((WorkflowPreConversionRel) rel).getWorkflow()); + setDSSToRelConversionRequestRef(rel.getDSSToRelConversionRequestRef()); + } + + @Override + public OrchestrationToRelConversionRequestRef getDSSToRelConversionRequestRef() { + return (OrchestrationToRelConversionRequestRef) super.getDSSToRelConversionRequestRef(); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerTask.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerTask.java new file mode 100644 index 000000000..982d5e408 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerTask.java @@ -0,0 +1,67 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity; + +import java.util.List; + +public class DolphinSchedulerTask { + + private String type; + + private String id; + + private String name; + + private DolphinSchedulerTaskParam params; + + private String description; + + private List preTasks; + + public void setType(String type) { + this.type = type; + } + + public String getType() { + return this.type; + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return this.id; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + public void setParams(DolphinSchedulerTaskParam params) { + this.params = params; + } + + public DolphinSchedulerTaskParam getParams() { + return this.params; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDescription() { + return this.description; + } + + public void setPreTasks(List preTasks) { + this.preTasks = preTasks; + } + + public List getPreTasks() { + return this.preTasks; + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerTaskParam.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerTaskParam.java new file mode 100644 index 000000000..1f539c8e8 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerTaskParam.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity; + + +public class DolphinSchedulerTaskParam { + + private String rawScript; + + public String getRawScript() { + return rawScript; + } + + public void setRawScript(String rawScript) { + this.rawScript = rawScript; + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerWorkflow.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerWorkflow.java new file mode 100644 index 000000000..e7ea4f048 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/entity/DolphinSchedulerWorkflow.java @@ -0,0 +1,161 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; + +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowWithContextImpl; + +public class DolphinSchedulerWorkflow extends WorkflowWithContextImpl { + + private ProcessDefinitionJson processDefinitionJson; + + private Map locations; + + private List connects; + + private String releaseState; + + public ProcessDefinitionJson getProcessDefinitionJson() { + return processDefinitionJson; + } + + public void setProcessDefinitionJson(ProcessDefinitionJson processDefinitionJson) { + this.processDefinitionJson = processDefinitionJson; + } + + public Map getLocations() { + return locations; + } + + public void setLocations(Map locations) { + this.locations = locations; + } + + public List getConnects() { + return connects; + } + + public void setConnects(List connects) { + this.connects = connects; + } + + public String getReleaseState() { + return releaseState; + } + + public void setReleaseState(String releaseState) { + this.releaseState = releaseState; + } + + public static class ProcessDefinitionJson { + private List> globalParams = new ArrayList<>(0); + private List tasks; + + public List getTasks() { + return tasks; + } + + public void setTasks(List tasks) { + this.tasks = tasks; + } + + public void addTask(DolphinSchedulerTask task) { + if (CollectionUtils.isEmpty(tasks)) { + tasks = new LinkedList<>(); + } + tasks.add(task); + } + public List> getGlobalParams() { + return globalParams; + } + + public void setGlobalParams(List> globalParams) { + this.globalParams = globalParams; + } + } + + public static class LocationInfo { + private String name; + + private String targetarr; + + // 后继节点数 + // private String nodenumber; + + private int x; + + private int y; + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + public void setTargetarr(String targetarr) { + this.targetarr = targetarr; + } + + public String getTargetarr() { + return this.targetarr; + } + + // public void setNodenumber(String nodenumber) { + // this.nodenumber = nodenumber; + // } + // + // public String getNodenumber() { + // return this.nodenumber; + // } + + public void setX(int x) { + this.x = x; + } + + public int getX() { + return this.x; + } + + public void setY(int y) { + this.y = y; + } + + public int getY() { + return this.y; + } + } + + public static class Connect { + private String endPointSourceId; + + private String endPointTargetId; + + public Connect(String endPointSourceId, String endPointTargetId) { + this.endPointSourceId = endPointSourceId; + this.endPointTargetId = endPointTargetId; + } + + public void setEndPointSourceId(String endPointSourceId) { + this.endPointSourceId = endPointSourceId; + } + + public String getEndPointSourceId() { + return this.endPointSourceId; + } + + public void setEndPointTargetId(String endPointTargetId) { + this.endPointTargetId = endPointTargetId; + } + + public String getEndPointTargetId() { + return this.endPointTargetId; + } + + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/linkisjob/LinkisJob.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/linkisjob/LinkisJob.java new file mode 100644 index 000000000..1e103eb22 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/linkisjob/LinkisJob.java @@ -0,0 +1,88 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.linkisjob; + +import java.util.Map; + +public class LinkisJob { + private String name; + private String type; + private String linkistype; + private String proxyUser; + private String dependencies; + private Map conf; + private String command; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getLinkistype() { + return linkistype; + } + + public void setLinkistype(String linkistype) { + this.linkistype = linkistype; + } + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + + public String getDependencies() { + return dependencies; + } + + public void setDependencies(String dependencies) { + this.dependencies = dependencies; + } + + public Map getConf() { + return conf; + } + + public void setConf(Map conf) { + this.conf = conf; + } + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectCreationOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectCreationOperation.java new file mode 100644 index 000000000..d9945e97d --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectCreationOperation.java @@ -0,0 +1,64 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.service.DolphinSchedulerProjectService; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.ProjectUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectCreationOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; +import java.util.Map; + + +public class DolphinSchedulerProjectCreationOperation + extends AbstractStructureOperation + implements ProjectCreationOperation { + + private String projectUrl; + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.projectUrl = mergeUrl(baseUrl, "projects/create"); + } + + @Override + public ProjectResponseRef createProject(DSSProjectContentRequestRef.DSSProjectContentRequestRefImpl requestRef) throws ExternalOperationFailedException { + // Dolphin Scheduler项目名 + String dsProjectName = ProjectUtils.generateDolphinProjectName(requestRef.getWorkspace().getWorkspaceName(), requestRef.getDSSProject().getName()); + logger.info("begin to create project in DolphinScheduler, project name is {}, creator is {}.", dsProjectName, requestRef.getDSSProject().getUsername()); + Map formData = MapUtils.newCommonMap("projectName", dsProjectName, "description", requestRef.getDSSProject().getDescription()); + DolphinSchedulerHttpUtils.getHttpPostResult(ssoRequestOperation, projectUrl, DolphinSchedulerConf.DS_ADMIN_USER.getValue(), formData); + RefProjectContentRequestRef searchRequestRef = new RefProjectContentRequestRef.RefProjectContentRequestRefImpl(); + searchRequestRef.setProjectName(dsProjectName).setWorkspace(requestRef.getWorkspace()) + .setUserName(DolphinSchedulerConf.DS_ADMIN_USER.getValue()); + Long refProjectId = ((ProjectService) service).getProjectSearchOperation().searchProject(searchRequestRef).getRefProjectId(); + logger.info("the refProjectId in dolphinScheduler of projectName:{} is:{}", requestRef.getDSSProject().getName(), refProjectId); + // 对releaseUsers授权工程可访问权限 + if(CollectionUtils.isNotEmpty(requestRef.getDSSProjectPrivilege().getReleaseUsers())) { + ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl updateRequestRef = new ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl(); + updateRequestRef.setAddedDSSProjectPrivilege(requestRef.getDSSProjectPrivilege()).setUserName(requestRef.getUserName()) + .setRefProjectId(refProjectId).setDSSProject(requestRef.getDSSProject()).setWorkspace(requestRef.getWorkspace()); + ((DolphinSchedulerProjectService) service).getProjectGrantOperation().grantProject(updateRequestRef); + } + // 返回project id + return ProjectResponseRef.newExternalBuilder().setRefProjectId(refProjectId).success(); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectDeletionOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectDeletionOperation.java new file mode 100644 index 000000000..9bff5454e --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectDeletionOperation.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.ProjectUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public class DolphinSchedulerProjectDeletionOperation + extends AbstractStructureOperation + implements ProjectDeletionOperation { + + private String deleteProjectByIdUrl; + + public DolphinSchedulerProjectDeletionOperation() {} + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.deleteProjectByIdUrl = mergeUrl(baseUrl, "projects/delete"); + } + + @Override + public ResponseRef deleteProject(RefProjectContentRequestRef.RefProjectContentRequestRefImpl projectRef) throws ExternalOperationFailedException { + // Dolphin Scheduler项目名 + String dsProjectName = + ProjectUtils.generateDolphinProjectName(projectRef.getWorkspace().getWorkspaceName(), projectRef.getProjectName()); + logger.info("User {} begin to delete project in DolphinScheduler, project name is {}.", projectRef.getUserName(), dsProjectName); + DolphinSchedulerHttpUtils.getHttpGetResult(ssoRequestOperation, this.deleteProjectByIdUrl, DolphinSchedulerConf.DS_ADMIN_USER.getValue(), + MapUtils.newCommonMap("projectId", projectRef.getRefProjectId())); + return ProjectResponseRef.newExternalBuilder().success(); + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectGrantOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectGrantOperation.java new file mode 100644 index 000000000..f6eac5c6d --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectGrantOperation.java @@ -0,0 +1,94 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref.DolphinSchedulerDataResponseRef; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso.DolphinSchedulerTokenManager; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectPrivilege; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +public class DolphinSchedulerProjectGrantOperation + extends AbstractStructureOperation { + + private String grantProjectUrl; + + private String authedProjectUrl; + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.grantProjectUrl = mergeUrl(baseUrl, "users/grant-project"); + this.authedProjectUrl = mergeUrl(baseUrl, "projects/authed-project"); + } + + + /** + * Grant project. + * + * @throws ExternalOperationFailedException + * the external operation failed exception + */ + public ResponseRef grantProject(ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl requestRef) + throws ExternalOperationFailedException { + BiConsumer, String>> dealPrivilege = (privilege, authedProjectIdsConsumer) -> { + if(privilege == null || CollectionUtils.isEmpty(privilege.getReleaseUsers())) { + return; + } + privilege.getReleaseUsers().forEach(releaseUser -> { + int userId = DolphinSchedulerTokenManager.getDolphinSchedulerTokenManager(getBaseUrl()).getUserId(releaseUser); + List authedProjectIds = getAuthedProjectIds(userId); + int originSize = authedProjectIds.size(); + authedProjectIdsConsumer.accept(authedProjectIds, releaseUser); + if(originSize != authedProjectIds.size()) { + grantTo(userId, authedProjectIds); + } + }); + }; + // 如果待授权用户不存在,则新建 + dealPrivilege.accept(requestRef.getAddedDSSProjectPrivilege(), (authedProjectIds, releaseUser) -> { + if (!authedProjectIds.contains(requestRef.getRefProjectId())) { + logger.info("try to grant access permission on project {} to user {}.", requestRef.getProjectName(), releaseUser); + authedProjectIds.add(requestRef.getRefProjectId()); + } + }); + // 被去掉的已授权用户,需清除 + dealPrivilege.accept(requestRef.getRemovedDSSProjectPrivilege(), (authedProjectIds, releaseUser) -> { + if(authedProjectIds.contains(requestRef.getRefProjectId())) { + logger.info("try to un-grant access permission on project {} to user {}.", requestRef.getProjectName(), releaseUser); + authedProjectIds.remove(requestRef.getRefProjectId()); + } + }); + return ResponseRef.newExternalBuilder().success(); + } + + private void grantTo(int userId, List authProjectIds) { + Map formData = MapUtils.newCommonMap("userId", userId, "projectIds", StringUtils.join(authProjectIds, ",")); + DolphinSchedulerHttpUtils.getHttpPostResult(ssoRequestOperation, grantProjectUrl, DolphinSchedulerConf.DS_ADMIN_USER.getValue(), formData); + } + + private List getAuthedProjectIds(int userId) throws ExternalOperationFailedException { + String url = this.authedProjectUrl + "?userId=" + userId; + DolphinSchedulerDataResponseRef responseRef = DolphinSchedulerHttpUtils.getHttpGetResult(ssoRequestOperation, url, DolphinSchedulerConf.DS_ADMIN_USER.getValue()); + List> projectList = responseRef.getData(); + logger.info("the authed project is: {}", projectList); + return projectList.stream().map(project -> DolphinSchedulerHttpUtils.parseToLong(project.get("id"))).collect(Collectors.toList()); + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectSearchOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectSearchOperation.java new file mode 100644 index 000000000..d501ec49b --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectSearchOperation.java @@ -0,0 +1,41 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref.DolphinSchedulerPageInfoResponseRef; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectSearchOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public class DolphinSchedulerProjectSearchOperation + extends AbstractStructureOperation + implements ProjectSearchOperation { + + private String listPagingUrl; + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.listPagingUrl = mergeUrl(baseUrl, "projects/list-paging"); + } + + @Override + public ProjectResponseRef searchProject(RefProjectContentRequestRef.RefProjectContentRequestRefImpl requestRef) throws ExternalOperationFailedException { + String url = this.listPagingUrl + "?pageNo=1&pageSize=40&searchVal=" + requestRef.getProjectName(); + DolphinSchedulerPageInfoResponseRef responseRef = DolphinSchedulerHttpUtils.getHttpGetResult(ssoRequestOperation, url, requestRef.getUserName()); + return responseRef.getTotalList().stream().filter(project -> requestRef.getProjectName().equals(project.get("name"))) + .map(project -> { + Long id = DolphinSchedulerHttpUtils.parseToLong(project.get("id")); + return ProjectResponseRef.newExternalBuilder().setRefProjectId(id).success(); + }).findAny().orElse(ProjectResponseRef.newExternalBuilder().success()); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectUpdateOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectUpdateOperation.java new file mode 100644 index 000000000..ceb82a7cc --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerProjectUpdateOperation.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.service.DolphinSchedulerProjectService; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.ProjectUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +import java.util.Map; + +public class DolphinSchedulerProjectUpdateOperation + extends AbstractStructureOperation + implements ProjectUpdateOperation { + + private String projectUpdateUrl; + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.projectUpdateUrl = mergeUrl(baseUrl, "projects/update"); + } + + @Override + public ProjectResponseRef updateProject(ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl requestRef) throws ExternalOperationFailedException { + // Dolphin Scheduler项目名 + String dsProjectName = + ProjectUtils.generateDolphinProjectName(requestRef.getWorkspace().getWorkspaceName(), + requestRef.getProjectName()); + logger.info("user {} begin to update project in DolphinScheduler, project name is {}.", requestRef.getUserName(), dsProjectName); + Map formData = MapUtils.newCommonMapBuilder().put("projectId", requestRef.getRefProjectId()) + .put("projectName", dsProjectName).put("description", requestRef.getDSSProject().getDescription()).build(); + DolphinSchedulerHttpUtils.getHttpPostResult(ssoRequestOperation, projectUpdateUrl, DolphinSchedulerConf.DS_ADMIN_USER.getValue(), formData); + // 更新授权用户 + ((DolphinSchedulerProjectService) service).getProjectGrantOperation().grantProject(requestRef); + return ProjectResponseRef.newExternalBuilder().success(); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerTokenGetOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerTokenGetOperation.java new file mode 100644 index 000000000..68edc655e --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerTokenGetOperation.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso.DolphinSchedulerTokenManager; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.optional.AbstractOptionalOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; + +/** + * @author enjoyyin + * @date 2022-03-18 + * @since 1.1.0 + */ +public class DolphinSchedulerTokenGetOperation extends AbstractOptionalOperation { + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } + + @Override + public String getOperationName() { + return "getToken"; + } + + @Override + public ResponseRef apply(StructureRequestRef ref) { + String token = DolphinSchedulerTokenManager.getDolphinSchedulerTokenManager(getBaseUrl()).getToken(ref.getUserName()); + long expireTime = DolphinSchedulerTokenManager.getDolphinSchedulerTokenManager(getBaseUrl()).getTokenExpireTime(ref.getUserName()); + return ResponseRef.newExternalBuilder().setResponseMap(MapUtils.newCommonMap("token", token, "expireTime", expireTime)).success(); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowCreationOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowCreationOperation.java new file mode 100644 index 000000000..e98f51d90 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowCreationOperation.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.ProjectUtils; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationCreationOperation; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationService; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.DSSOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationResponseRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.RefOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class DolphinSchedulerWorkflowCreationOperation + extends AbstractStructureOperation + implements OrchestrationCreationOperation { + + private String createProcessDefinitionByIdUrl; + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.createProcessDefinitionByIdUrl = mergeUrl(baseUrl, "projects/${projectName}/process/save"); + } + + @Override + public OrchestrationResponseRef createOrchestration(DSSOrchestrationContentRequestRef.DSSOrchestrationContentRequestRefImpl orchestrationRef) throws ExternalOperationFailedException { + String dolphinProjectName = + ProjectUtils.generateDolphinProjectName(orchestrationRef.getWorkspace().getWorkspaceName(), orchestrationRef.getProjectName()); + logger.info("begin to create workflow in DolphinScheduler, project name is {}, workflow name is {}, creator is {}.", + dolphinProjectName, orchestrationRef.getDSSOrchestration().getName(), orchestrationRef.getUserName()); + String createUrl = + StringUtils.replace(this.createProcessDefinitionByIdUrl, "${projectName}", dolphinProjectName); + Map formData = MapUtils.newCommonMapBuilder().put("name", orchestrationRef.getDSSOrchestration().getName()) + .put("description", orchestrationRef.getDSSOrchestration().getDescription()) + .put("processDefinitionJson", "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"DSS_INIT_EMPTY_NODE\",\"name\":\"init_empty_node\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"This node is only used for DSS to create the workflow, when a publishment is called by DSS, this workflow will be updated by DSS.\\\"\"},\"description\":\"\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"runFlag\":\"NORMAL\",\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"taskInstancePriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}") + .put("locations", "{\"DSS_INIT_EMPTY_NODE\":{\"name\":\"init_empty_node\",\"targetarr\":\"\",\"nodenumber\":\"0\",\"x\":236,\"y\":60}}") + .put("connects", "[]").build(); + DolphinSchedulerHttpUtils.getHttpPostResult(ssoRequestOperation, createUrl, orchestrationRef.getUserName(), formData); + // 获取id + RefOrchestrationContentRequestRef.RefOrchestrationContentRequestRefImpl ref = new RefOrchestrationContentRequestRef.RefOrchestrationContentRequestRefImpl() + .setRefProjectId(orchestrationRef.getRefProjectId()).setOrchestrationName(orchestrationRef.getDSSOrchestration().getName()) + .setProjectName(orchestrationRef.getProjectName()).setWorkspace(orchestrationRef.getWorkspace()) + .setUserName(orchestrationRef.getUserName()); + return ((OrchestrationService) service).getOrchestrationSearchOperation().searchOrchestration(ref); + } + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowDeletionOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowDeletionOperation.java new file mode 100644 index 000000000..146e64088 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowDeletionOperation.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.ProjectUtils; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationDeletionOperation; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.RefOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import org.apache.commons.lang3.StringUtils; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class DolphinSchedulerWorkflowDeletionOperation + extends AbstractStructureOperation + implements OrchestrationDeletionOperation { + + private String deleteProcessDefinitionByIdUrl; + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.deleteProcessDefinitionByIdUrl = mergeUrl(baseUrl, "projects/${projectName}/process/delete"); + } + + @Override + public ResponseRef deleteOrchestration(RefOrchestrationContentRequestRef.RefOrchestrationContentRequestRefImpl requestRef) { + String dolphinProjectName = + ProjectUtils.generateDolphinProjectName(requestRef.getWorkspace().getWorkspaceName(), requestRef.getProjectName()); + String deleteUrl = + StringUtils.replace(this.deleteProcessDefinitionByIdUrl, "${projectName}", dolphinProjectName); + logger.info("begin to create workflow in DolphinScheduler, project name is {}, workflow name is {}, creator is {}.", + dolphinProjectName, requestRef.getOrchestrationName(), requestRef.getUserName()); + DolphinSchedulerHttpUtils.getHttpGetResult(ssoRequestOperation, deleteUrl, requestRef.getUserName(), MapUtils.newCommonMap("processDefinitionId", requestRef.getRefOrchestrationId())); + return ResponseRef.newExternalBuilder().success(); + } + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowSearchOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowSearchOperation.java new file mode 100644 index 000000000..b9e93e363 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowSearchOperation.java @@ -0,0 +1,85 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref.DolphinSchedulerDataResponseRef; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref.DolphinSchedulerPageInfoResponseRef; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.ProjectUtils; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationSearchOperation; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationResponseRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.RefOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; + +public class DolphinSchedulerWorkflowSearchOperation + extends AbstractStructureOperation + implements OrchestrationSearchOperation { + + private String getProcessDefinitionByIdUrl; + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.getProcessDefinitionByIdUrl = mergeUrl(baseUrl, "projects/${projectName}/process/list"); + } + + @Override + public OrchestrationResponseRef searchOrchestration(RefOrchestrationContentRequestRef.RefOrchestrationContentRequestRefImpl ref) throws ExternalOperationFailedException { + String dolphinProjectName = + ProjectUtils.generateDolphinProjectName(ref.getWorkspace().getWorkspaceName(), ref.getProjectName()); + String getUrl = StringUtils.replace(this.getProcessDefinitionByIdUrl, "${projectName}", dolphinProjectName); + DolphinSchedulerDataResponseRef responseRef = DolphinSchedulerHttpUtils.getHttpGetResult(ssoRequestOperation, getUrl, ref.getUserName()); + List> dataList = responseRef.getData(); + return dataList.stream().filter(workflow -> ref.getOrchestrationName().equals(workflow.get("name"))) + .map(workflow -> DolphinSchedulerHttpUtils.parseToLong(workflow.get("id"))).map(id -> OrchestrationResponseRef.newExternalBuilder().setRefOrchestrationId(id).success()) + .findAny().orElse(OrchestrationResponseRef.newExternalBuilder().success()); + } + +// private String queryProcessDefinitionReleaseStateById(String projectName, Long processId, String userName) +// throws ExternalOperationFailedException { +// String queryUrl = StringUtils.replace(this.verifyProcessDefinitionByIdUrl, "${projectName}", projectName); +// +// CloseableHttpResponse httpResponse = null; +// String entString = null; +// int httpStatusCode = 0; +// try { +// URIBuilder uriBuilder = new URIBuilder(queryUrl); +// uriBuilder.addParameter("processId", String.valueOf(processId)); +// DolphinSchedulerHttpGet httpGet = new DolphinSchedulerHttpGet(uriBuilder.build(), userName); +// +// httpResponse = this.getOperation.requestWithSSO(this.ssoUrlBuilderOperation, httpGet); +// +// HttpEntity ent = httpResponse.getEntity(); +// entString = IOUtils.toString(ent.getContent(), StandardCharsets.UTF_8); +// httpStatusCode = httpResponse.getStatusLine().getStatusCode(); +// } catch (final Exception e) { +// SchedulisExceptionUtils.dealErrorException(90021, "获取工作流调度状态失败", e, ExternalOperationFailedException.class); +// } finally { +// IOUtils.closeQuietly(httpResponse); +// } +// +// try { +// if (HttpStatus.SC_OK == httpStatusCode && DolphinAppConnUtils.getCodeFromEntity(entString) == 0) { +// ObjectMapper mapper = new ObjectMapper(); +// JsonNode jsonNode = mapper.readTree(entString); +// JsonNode dataNode = jsonNode.get("data"); +// return dataNode.get("releaseState").asText(); +// } +// } catch (IOException e) { +// throw new ExternalOperationFailedException(90022, "工作流调度状态解析失败", e); +// } +// logger.warn("Dolphin Scheduler上不存在该工作流定义:{}", entString); +// return null; +// } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowUpdateOperation.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowUpdateOperation.java new file mode 100644 index 000000000..1ec107c59 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/operation/DolphinSchedulerWorkflowUpdateOperation.java @@ -0,0 +1,80 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerWorkflow; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref.DolphinSchedulerDataResponseRef; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.ProjectUtils; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationUpdateOperation; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationUpdateRequestRef; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.utils.JsonUtils; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class DolphinSchedulerWorkflowUpdateOperation + extends AbstractStructureOperation +implements OrchestrationUpdateOperation { + + private String updateProcessDefinitionByIdUrl; + private String selectProcessDefinitionByIdUrl; + + @Override + public void init() { + super.init(); + String baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(getBaseUrl()); + this.updateProcessDefinitionByIdUrl = mergeUrl(baseUrl, "projects/${projectName}/process/update"); + this.selectProcessDefinitionByIdUrl = mergeUrl(baseUrl, "projects/${projectName}/process/select-by-id"); + } + + @Override + public ResponseRef updateOrchestration(OrchestrationUpdateRequestRef.OrchestrationUpdateRequestRefImpl orchestrationRef) throws ExternalOperationFailedException { + String dolphinProjectName = + ProjectUtils.generateDolphinProjectName(orchestrationRef.getWorkspace().getWorkspaceName(), orchestrationRef.getProjectName()); + logger.info("begin to update workflow in DolphinScheduler, project name is {}, workflow name is {}, creator is {}.", + dolphinProjectName, orchestrationRef.getDSSOrchestration().getName(), orchestrationRef.getUserName()); + String updateUrl = + StringUtils.replace(this.updateProcessDefinitionByIdUrl, "${projectName}", dolphinProjectName); + String processDefinitionJson, locations, connects; + if(orchestrationRef.getDSSOrchestration() instanceof DolphinSchedulerWorkflow) { + DolphinSchedulerWorkflow workflow = (DolphinSchedulerWorkflow) orchestrationRef.getDSSOrchestration(); + try { + processDefinitionJson = JsonUtils.jackson().writeValueAsString(workflow.getProcessDefinitionJson()); + locations = JsonUtils.jackson().writeValueAsString(workflow.getLocations()); + connects = JsonUtils.jackson().writeValueAsString(workflow.getConnects()); + } catch (JsonProcessingException e) { + throw new ExternalOperationFailedException(90321, "parse workflow object to DolphinScheduler workflow string failed."); + } + } else { + String selectUrl = StringUtils.replace(selectProcessDefinitionByIdUrl, "${projectName}", dolphinProjectName); + DolphinSchedulerDataResponseRef responseRef = DolphinSchedulerHttpUtils.getHttpGetResult(ssoRequestOperation, selectUrl + "?processId=" + + orchestrationRef.getRefOrchestrationId(), orchestrationRef.getUserName()); + Map workflow = responseRef.getData(); + processDefinitionJson = (String) workflow.get("processDefinitionJson"); + locations = (String) workflow.get("locations"); + connects = (String) workflow.get("connects"); + } + Map formData = MapUtils.newCommonMapBuilder().put("name", orchestrationRef.getDSSOrchestration().getName()) + .put("description", orchestrationRef.getDSSOrchestration().getDescription()) + .put("id", orchestrationRef.getRefOrchestrationId()) + .put("processDefinitionJson", processDefinitionJson) + .put("locations", locations).put("connects", connects).build(); + DolphinSchedulerHttpUtils.getHttpPostResult(ssoRequestOperation, updateUrl, orchestrationRef.getUserName(), formData); + return ResponseRef.newExternalBuilder().success(); + } + + @Override + protected String getAppConnName() { + return DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME; + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerDataResponseRef.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerDataResponseRef.java new file mode 100644 index 000000000..ea4ceae49 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerDataResponseRef.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class DolphinSchedulerDataResponseRef extends ResponseRefImpl { + + private Object data; + + public DolphinSchedulerDataResponseRef(String responseBody, int status, + String errorMsg, + Map responseMap, + Object data) { + super(responseBody, status, errorMsg, responseMap); + this.data = data; + } + + public T getData() { + return (T) data; + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerPageInfoResponseRef.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerPageInfoResponseRef.java new file mode 100644 index 000000000..48323f0fe --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerPageInfoResponseRef.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class DolphinSchedulerPageInfoResponseRef extends ResponseRefImpl { + public DolphinSchedulerPageInfoResponseRef(String responseBody, int status, String errorMsg, Map responseMap) { + super(responseBody, status, errorMsg, responseMap); + } + + public int getTotalPage() { + return (int) toMap().get("totalPage"); + } + + public int getCurrentPage() { + return (int) toMap().get("currentPage"); + } + + public int getTotal() { + return (int) toMap().get("total"); + } + + public List> getTotalList() { + if(!toMap().containsKey("totalList") || toMap().get("totalList") == null) { + return new ArrayList<>(0); + } + return (List>) toMap().get("totalList"); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerResponseRefBuilder.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerResponseRefBuilder.java new file mode 100644 index 000000000..f5f82c28b --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/ref/DolphinSchedulerResponseRefBuilder.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.constant.Constant; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 1.0.0 + */ +public class DolphinSchedulerResponseRefBuilder + extends ResponseRefBuilder.ExternalResponseRefBuilder { + + public static DolphinSchedulerResponseRefBuilder newBuilder() { + return new DolphinSchedulerResponseRefBuilder(); + } + + private Object data; + + @Override + public DolphinSchedulerResponseRefBuilder setResponseBody(String responseBody) { + Map responseMap = DSSCommonUtils.COMMON_GSON.fromJson(responseBody, Map.class); + status = (int) DolphinSchedulerHttpUtils.parseToLong(responseMap.get("code")); + if(status != Constant.DS_RESULT_CODE_SUCCESS) { + errorMsg = (String) responseMap.get("msg"); + throw new ExternalOperationFailedException(90051, "request to DolphinScheduler failed. Caused by: " + errorMsg); + } + Object data = responseMap.get("data"); + if(data instanceof Map && ((Map) data).containsKey("totalList")) { + setResponseMap((Map) data); + } else { + this.data = data; + } + return super.setResponseBody(responseBody); + } + + @Override + protected ResponseRefImpl createResponseRef() { + if(data != null) { + return new DolphinSchedulerDataResponseRef(responseBody, status, errorMsg, responseMap, data); + } + return new DolphinSchedulerPageInfoResponseRef(responseBody, status, errorMsg, responseMap); + } +} + + diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/service/DolphinSchedulerOrchestrationService.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/service/DolphinSchedulerOrchestrationService.java new file mode 100644 index 000000000..7d606598f --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/service/DolphinSchedulerOrchestrationService.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.service; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation.DolphinSchedulerWorkflowCreationOperation; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation.DolphinSchedulerWorkflowDeletionOperation; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation.DolphinSchedulerWorkflowSearchOperation; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation.DolphinSchedulerWorkflowUpdateOperation; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso.DolphinSchedulerTokenManager; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.*; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class DolphinSchedulerOrchestrationService extends OrchestrationService { + + @Override + public void setAppInstance(AppInstance appInstance) { + super.setAppInstance(appInstance); + DolphinSchedulerTokenManager.getDolphinSchedulerTokenManager(appInstance.getBaseUrl()) + .setSSORequestOperation(getSSORequestService() + .createSSORequestOperation(DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME)); + } + + @Override + protected OrchestrationCreationOperation createOrchestrationCreationOperation() { + return new DolphinSchedulerWorkflowCreationOperation(); + } + + @Override + protected OrchestrationUpdateOperation createOrchestrationUpdateOperation() { + return new DolphinSchedulerWorkflowUpdateOperation(); + } + + @Override + protected OrchestrationDeletionOperation createOrchestrationDeletionOperation() { + return new DolphinSchedulerWorkflowDeletionOperation(); + } + + @Override + protected OrchestrationSearchOperation createOrchestrationSearchOperation() { + return new DolphinSchedulerWorkflowSearchOperation(); + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/service/DolphinSchedulerProjectService.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/service/DolphinSchedulerProjectService.java new file mode 100644 index 000000000..78fc7bf29 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/service/DolphinSchedulerProjectService.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.service; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.operation.*; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso.DolphinSchedulerTokenManager; +import com.webank.wedatasphere.dss.standard.app.structure.project.*; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +public class DolphinSchedulerProjectService extends ProjectService { + + @Override + public void setAppInstance(AppInstance appInstance) { + super.setAppInstance(appInstance); + DolphinSchedulerTokenManager.getDolphinSchedulerTokenManager(appInstance.getBaseUrl()) + .setSSORequestOperation(getSSORequestService() + .createSSORequestOperation(DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME)); + } + + @Override + protected ProjectCreationOperation createProjectCreationOperation() { + return new DolphinSchedulerProjectCreationOperation(); + } + + @Override + protected ProjectUpdateOperation createProjectUpdateOperation() { + return new DolphinSchedulerProjectUpdateOperation(); + } + + @Override + protected ProjectDeletionOperation createProjectDeletionOperation() { + return new DolphinSchedulerProjectDeletionOperation(); + } + + @Override + protected ProjectSearchOperation createProjectSearchOperation() { + return new DolphinSchedulerProjectSearchOperation(); + } + + public DolphinSchedulerProjectGrantOperation getProjectGrantOperation() { + return this.getOrCreate(DolphinSchedulerProjectGrantOperation::new, DolphinSchedulerProjectGrantOperation.class); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/AbstractDolphinSchedulerTokenManager.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/AbstractDolphinSchedulerTokenManager.java new file mode 100644 index 000000000..b670fbcc1 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/AbstractDolphinSchedulerTokenManager.java @@ -0,0 +1,170 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerAccessToken; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.standard.common.utils.AppStandardClassUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.linkis.common.utils.ByteTimeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf.DS_TOKEN_EXPIRE_TIME; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public abstract class AbstractDolphinSchedulerTokenManager implements DolphinSchedulerTokenManager { + + // 一个DSS系统,只支持对接一个调度系统,所以这里只需要给出一个实例即可 + static DolphinSchedulerTokenManager dolphinSchedulerTokenManager; + static final List dolphinSchedulerTokenManagers = + AppStandardClassUtils.getInstance(DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME).getInstances(DolphinSchedulerTokenManager.class); + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + protected UserCreationFactory userCreationFactory; + protected final Map userTokens = new ConcurrentHashMap<>(); + protected String baseUrl; + protected SSORequestOperation ssoRequestOperation; + + @Override + public String getBaseUrl() { + return baseUrl; + } + + @Override + public void init(String baseUrl) { + if(StringUtils.isBlank(DolphinSchedulerConf.DS_ADMIN_TOKEN.getValue())) { + logger.error("please set {} for DolphinScheduler AppConn.", DolphinSchedulerConf.DS_ADMIN_TOKEN.key()); + throw new ExternalOperationFailedException(90388, "please set " + DolphinSchedulerConf.DS_ADMIN_TOKEN.key() + " for DolphinScheduler AppConn."); + } + this.baseUrl = DolphinSchedulerHttpUtils.getDolphinSchedulerBaseUrl(baseUrl); + userCreationFactory = AppStandardClassUtils.getInstance(DolphinSchedulerAppConn.DOLPHINSCHEDULER_APPCONN_NAME) + .getInstanceOrDefault(UserCreationFactory.class, new UserCreationFactory()); + logger.info("use {} to create new DolphinScheduler users.", userCreationFactory.getClass().getSimpleName()); + } + + @Override + public void setSSORequestOperation(SSORequestOperation ssoRequestOperation) { + this.ssoRequestOperation = ssoRequestOperation; + } + + @Override + public int getUserId(String userName) { + if(userTokens.containsKey(userName)) { + return userTokens.get(userName).getUserId(); + } + synchronized (userName.intern()) { + if(userTokens.containsKey(userName)) { + return userTokens.get(userName).getUserId(); + } + Integer userId = fetchUserId(userName); + if(userId == null) { + // 用户不存在,创建该用户 + createUser(userName); + userId = fetchUserId(userName); + } + return userId; + } + } + + @Override + public long getTokenExpireTime(String userName) { + if (DolphinSchedulerConf.DS_ADMIN_USER.getValue().equals(userName)) { + return System.currentTimeMillis() + ByteTimeUtils.timeStringAsMs("24h"); + } + if(userTokens.containsKey(userName)) { + return userTokens.get(userName).getExpireTime().getTime(); + } + getToken(userName); + return userTokens.get(userName).getExpireTime().getTime(); + } + + @Override + public final String getToken(String userName) { + if (DolphinSchedulerConf.DS_ADMIN_USER.getValue().equals(userName)) { + return DolphinSchedulerConf.DS_ADMIN_TOKEN.getValue(); + } + if(userTokens.containsKey(userName)) { + DolphinSchedulerAccessToken userToken = userTokens.get(userName); + if(userToken.getExpireTime().getTime() - System.currentTimeMillis() <= DolphinSchedulerConf.DS_TOKEN_EXPIRE_TIME_GAP.getValue().toLong()) { + // 刷新token + updateToken(userToken, getNewExpireTime()); + } + return userToken.getToken(); + } + DolphinSchedulerAccessToken userToken = null; + Integer userId; + synchronized (userName.intern()) { + userId = fetchUserId(userName); + if(userId == null) { + // 用户不存在,创建该用户 + createUser(userName); + userId = fetchUserId(userName); + } else { + // 如果是已经存在的用户,可能已经有token了 + userToken = getTokenByUserName(userName, userId); + } + if(userToken != null) { + // 已有token未过期,返回该token + userToken.setUserName(userName); + userTokens.put(userName, userToken); + return userToken.getToken(); + } + // 该用户没有相应的token,执行创建操作 + String expireTime = getNewExpireTime(); + createToken(userId, expireTime); + } + userToken = getTokenByUserName(userName, userId); + if(userToken == null) { + throw new ExternalOperationFailedException(90321, "cannot find the token from DolphinScheduler of user " + userName); + } + userToken.setUserName(userName); + userTokens.put(userName, userToken); + return userToken.getToken(); + } + + private String getNewExpireTime() { + long expireTime = System.currentTimeMillis() + DS_TOKEN_EXPIRE_TIME.getValue().toLong(); + return DateFormatUtils.format(new Date(expireTime), "yyyy-MM-dd HH:mm:ss"); + } + + protected T getHttpGetResult(String url) { + return DolphinSchedulerHttpUtils.getHttpGetResult(ssoRequestOperation, url, DolphinSchedulerConf.DS_ADMIN_USER.getValue()); + } + + protected T getHttpPostResult(String url, Map formData) { + return DolphinSchedulerHttpUtils.getHttpPostResult(ssoRequestOperation, url, DolphinSchedulerConf.DS_ADMIN_USER.getValue(), formData); + } + + protected T getHttpPutResult(String url, Map formData) { + return DolphinSchedulerHttpUtils.getHttpPutResult(ssoRequestOperation, url, DolphinSchedulerConf.DS_ADMIN_USER.getValue(), formData); + } + + protected abstract void createUser(String userName) throws ExternalOperationFailedException; + + protected abstract Integer fetchUserId(String userName) throws ExternalOperationFailedException; + + protected abstract DolphinSchedulerAccessToken getTokenByUserName(String userName, int userId) + throws ExternalOperationFailedException; + + protected abstract String createToken(int userId, String expireTime) + throws ExternalOperationFailedException; + + protected abstract void updateToken(DolphinSchedulerAccessToken userToken, String expireTime) + throws ExternalOperationFailedException; + + protected abstract String generateToken(int userId, String expireTime) throws ExternalOperationFailedException; +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerSSOIntegrationStandardFactory.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerSSOIntegrationStandardFactory.java new file mode 100644 index 000000000..65c9ca723 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerSSOIntegrationStandardFactory.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso; + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandardFactory; +import com.webank.wedatasphere.dss.standard.app.sso.origin.HttpSSOIntegrationStandard; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 1.1.0 + */ +public class DolphinSchedulerSSOIntegrationStandardFactory implements SSOIntegrationStandardFactory { + + private SSOIntegrationStandard ssoIntegrationStandard; + private Logger logger = LoggerFactory.getLogger(DolphinSchedulerSSOIntegrationStandardFactory.class); + + @Override + public void init() { + ssoIntegrationStandard = new HttpSSOIntegrationStandard(); + logger.info("DolphinScheduler AppConn will use {} to integrate with DSS in 1st SSO standard.", ssoIntegrationStandard.getClass().getName()); + } + + @Override + public SSOIntegrationStandard getSSOIntegrationStandard() { + return ssoIntegrationStandard; + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerTokenManager.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerTokenManager.java new file mode 100644 index 000000000..3ff428068 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerTokenManager.java @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf; +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public interface DolphinSchedulerTokenManager { + + void init(String baseUrl); + + void setSSORequestOperation(SSORequestOperation ssoRequestOperation); + + /** + * 该 TokenManager 是否可以兼容 dsVersion 所传入的 DolphinScheduler 版本 + * @param dsVersion DolphinScheduler 版本 + * @return 如果兼容,返回true + */ + boolean isCompatible(String dsVersion); + + String getToken(String userName); + + long getTokenExpireTime(String userName); + + int getUserId(String userName); + + String getBaseUrl(); + + static DolphinSchedulerTokenManager getDolphinSchedulerTokenManager(String url) { + // 一个DSS系统,只支持对接一个调度系统,所以这里只需要给出一个实例即可 + if(AbstractDolphinSchedulerTokenManager.dolphinSchedulerTokenManager != null) { + return AbstractDolphinSchedulerTokenManager.dolphinSchedulerTokenManager; + } + synchronized (DolphinSchedulerTokenManager.class) { + if(AbstractDolphinSchedulerTokenManager.dolphinSchedulerTokenManager != null) { + return AbstractDolphinSchedulerTokenManager.dolphinSchedulerTokenManager; + } + String baseUrl = HttpClient.getBaseUrl(url); + AbstractDolphinSchedulerTokenManager.dolphinSchedulerTokenManager = AbstractDolphinSchedulerTokenManager.dolphinSchedulerTokenManagers.stream() + .filter(tokenManager -> tokenManager.isCompatible(DolphinSchedulerConf.DS_VERSION.getValue())) + .findAny().orElseThrow(() -> new ExternalOperationFailedException(90304, "Cannot find a suitable DolphinSchedulerTokenManager for DolphinScheduler version " + + DolphinSchedulerConf.DS_VERSION.getValue())); + AbstractDolphinSchedulerTokenManager.dolphinSchedulerTokenManager.init(baseUrl); + } + return AbstractDolphinSchedulerTokenManager.dolphinSchedulerTokenManager; + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerTokenManager_1_X.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerTokenManager_1_X.java new file mode 100644 index 000000000..3174227e2 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/DolphinSchedulerTokenManager_1_X.java @@ -0,0 +1,167 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.entity.DolphinSchedulerAccessToken; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref.DolphinSchedulerDataResponseRef; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref.DolphinSchedulerPageInfoResponseRef; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils.DolphinSchedulerHttpUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.lang3.time.DateUtils; + +import java.text.ParseException; +import java.util.Map; +import java.util.Optional; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public class DolphinSchedulerTokenManager_1_X extends AbstractDolphinSchedulerTokenManager { + + private String verifyUserNameUrl; + + private String createUserUrl; + + private String queryUserPageUrl; + + private String queryAccessTokenListUrl; + + private String generateTokenUrl; + + private String createTokenUrl; + + private String updateTokenUrl; + + @Override + public void init(String baseUrl) { + super.init(baseUrl); + baseUrl = getBaseUrl(); + this.verifyUserNameUrl = + baseUrl.endsWith("/") ? baseUrl + "users/verify-user-name" : baseUrl + "/users/verify-user-name"; + this.createUserUrl = baseUrl.endsWith("/") ? baseUrl + "users/create" : baseUrl + "/users/create"; + this.queryUserPageUrl = baseUrl.endsWith("/") ? baseUrl + "users/list-paging" : baseUrl + "/users/list-paging"; + + this.queryAccessTokenListUrl = + baseUrl.endsWith("/") ? baseUrl + "access-token/list-paging" : baseUrl + "/access-token/list-paging"; + this.generateTokenUrl = + baseUrl.endsWith("/") ? baseUrl + "access-token/generate" : baseUrl + "/access-token/generate"; + this.createTokenUrl = + baseUrl.endsWith("/") ? baseUrl + "access-token/create" : baseUrl + "/access-token/create"; + this.updateTokenUrl = + baseUrl.endsWith("/") ? baseUrl + "access-token/update" : baseUrl + "/access-token/update"; + } + + @Override + public boolean isCompatible(String dsVersion) { + return dsVersion.startsWith("1.3"); + } + + /** + * 验证用户名是否可用. + * + * @param userName 用户名 + * @return true:该用户名不存在,可使用. + * @throws ExternalOperationFailedException 失败 + */ + private boolean verifyUserName(String userName) throws ExternalOperationFailedException { + String url = this.verifyUserNameUrl + "?userName=" + userName; + try { + getHttpGetResult(url); + } catch (Exception e) { + throw new ExternalOperationFailedException(90051, "DolphinScheduler 验证用户名失败.", e); + } + return true; + } + + /** + * DS中新建用户. + * + * @param userName 用户名 + * @throws ExternalOperationFailedException 失败 + */ + @Override + protected void createUser(String userName) throws ExternalOperationFailedException { + logger.info("Try to ask DolphinScheduler to create user {}.", userName); + UserCreationFactory.User user = userCreationFactory.createUser(userName); + Map formData = MapUtils.newCommonMapBuilder().put("userName", userName) + .put("userPassword", user.getUserPassword()) + .put("tenantId", user.getTenantId()) + .put("email", user.getEmail()) + .put("queue", user.getQueue()).build(); + getHttpPostResult(createUserUrl, formData); + } + + @Override + protected Integer fetchUserId(String userName) throws ExternalOperationFailedException { + String url = this.queryUserPageUrl + "?pageNo=1&pageSize=20&searchVal=" + userName; + logger.info("begin to fetch userId for user:{}, url is: {}", userName, url); + DolphinSchedulerPageInfoResponseRef responseRef = getHttpGetResult(url); + Optional userId = responseRef.getTotalList().stream() + .filter(user -> userName.equals(user.get("userName"))) + .findAny().map(user -> (int) DolphinSchedulerHttpUtils.parseToLong(user.get("id"))); + return userId.orElse(null); + } + + /** + * 查找用户的 accessToken,没有则返回 null. + * + * @param userName 用户名 + * @param userId 用户Id + * @return token + * @throws ExternalOperationFailedException 失败 + */ + @Override + protected DolphinSchedulerAccessToken getTokenByUserName(String userName, int userId) + throws ExternalOperationFailedException { + String url = this.queryAccessTokenListUrl + "?pageNo=1&pageSize=20&searchVal=" + userName; + DolphinSchedulerPageInfoResponseRef responseRef = getHttpGetResult(url); + Map tokenMap = responseRef.getTotalList().stream() + .filter(token -> userId == DolphinSchedulerHttpUtils.parseToLong(token.get("userId"))).findAny() + .orElse(null); + if(tokenMap == null) { + return null; + } + DolphinSchedulerAccessToken token = new DolphinSchedulerAccessToken(); + token.setUserName(userName); + token.setId((int) DolphinSchedulerHttpUtils.parseToLong(tokenMap.get("id"))); + token.setToken((String) tokenMap.get("token")); + token.setUserId((int) DolphinSchedulerHttpUtils.parseToLong(tokenMap.get("userId"))); + try { + token.setExpireTime(DateUtils.parseDate((String) tokenMap.get("expireTime"), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")); + } catch (ParseException e) { + throw new ExternalOperationFailedException(90321, "parse the date format of DolphinScheduler failed, date string is " + + tokenMap.get("expireTime"), e); + } + return token; + } + + @Override + protected String createToken(int userId, String expireTime) + throws ExternalOperationFailedException { + String token = generateToken(userId, expireTime); + Map formData = MapUtils.newCommonMapBuilder() + .put("userId", userId) + .put("expireTime", expireTime) + .put("token", token).build(); + getHttpPostResult(createTokenUrl, formData); + return token; + } + + @Override + protected void updateToken(DolphinSchedulerAccessToken userToken, String expireTime) + throws ExternalOperationFailedException { + Map formData = MapUtils.newCommonMapBuilder().put("id", userToken.getId()) + .put("userId", userToken.getUserId()) + .put("expireTime", expireTime) + .put("token", userToken.getToken()).build(); + getHttpPostResult(updateTokenUrl, formData); + } + + @Override + protected String generateToken(int userId, String expireTime) throws ExternalOperationFailedException { + Map formData = MapUtils.newCommonMap("userId", userId, "expireTime", expireTime); + DolphinSchedulerDataResponseRef responseRef = getHttpPostResult(generateTokenUrl, formData); + return responseRef.getData(); + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/UserCreationFactory.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/UserCreationFactory.java new file mode 100644 index 000000000..ac1dc1640 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/sso/UserCreationFactory.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso; + +import org.apache.commons.lang3.RandomStringUtils; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class UserCreationFactory { + + public User createUser(String userName) { + return new User() { + @Override + public String getUserName() { + return userName; + } + @Override + public String getUserPassword() { + return RandomStringUtils.random(8); + } + @Override + public String getTenantId() { + return "1"; + } + @Override + public String getEmail() { + return "xx@qq.com"; + } + @Override + public String getQueue() { + return "default"; + } + }; + } + + public interface User { + + String getUserName(); + + String getUserPassword(); + + String getTenantId(); + + String getEmail(); + + String getQueue(); + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/standard/DolphinSchedulerStructureStandard.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/standard/DolphinSchedulerStructureStandard.java new file mode 100644 index 000000000..a0da0d49a --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/standard/DolphinSchedulerStructureStandard.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.standard; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.service.DolphinSchedulerOrchestrationService; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.service.DolphinSchedulerProjectService; +import com.webank.wedatasphere.dss.appconn.scheduler.AbstractSchedulerStructureIntegrationStandard; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; + +public class DolphinSchedulerStructureStandard extends AbstractSchedulerStructureIntegrationStandard { + + private static volatile DolphinSchedulerStructureStandard instance; + + + private DolphinSchedulerStructureStandard() { + } + + public static DolphinSchedulerStructureStandard getInstance() { + if (instance == null) { + synchronized (DolphinSchedulerStructureStandard.class) { + if (instance == null) { + instance = new DolphinSchedulerStructureStandard(); + } + } + } + return instance; + } + + @Override + protected ProjectService createProjectService() { + return new DolphinSchedulerProjectService(); + } + + @Override + protected OrchestrationService createOrchestrationService() { + return new DolphinSchedulerOrchestrationService(); + } +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/utils/DolphinSchedulerHttpUtils.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/utils/DolphinSchedulerHttpUtils.java new file mode 100644 index 000000000..228f9e60f --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/utils/DolphinSchedulerHttpUtils.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils; + +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.conf.DolphinSchedulerConf; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.ref.DolphinSchedulerResponseRefBuilder; +import com.webank.wedatasphere.dss.appconn.dolphinscheduler.sso.DolphinSchedulerTokenManager; +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSGetAction; +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSHttpAction; +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSPostAction; +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSPutAction; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.http.HttpStatus; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-17 + * @since 0.5.0 + */ +public class DolphinSchedulerHttpUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(DolphinSchedulerHttpUtils.class); + + public static T getHttpResult(SSORequestOperation ssoRequestOperation, DSSHttpAction action) { + HttpResult httpResult; + try { + httpResult = (HttpResult) ssoRequestOperation.requestWithSSO(null, action); + } catch (Exception e) { + LOGGER.error("user {} send request to dolphinscheduler in url {} failed, the requestBody is {}.", action.getUser(), action.getURL(), action.getRequestBody()); + throw new ExternalOperationFailedException(90322, "send request to dolphinscheduler failed. Caused by: " + ExceptionUtils.getRootCauseMessage(e), e); + } + String responseBody = httpResult.getResponseBody(); + int httpStatusCode = httpResult.getStatusCode(); + if (HttpStatus.SC_OK != httpStatusCode && HttpStatus.SC_CREATED != httpStatusCode) { + throw new ExternalOperationFailedException(90051, "request to DolphinScheduler with url " + action.getURL() + + " for user " + action.getUser() + " failed, the response http status code is " + httpStatusCode); + } + return (T) DolphinSchedulerResponseRefBuilder.newBuilder().setResponseBody(responseBody).build(); + } + + public static T getHttpResult(SSORequestOperation ssoRequestOperation, DSSHttpAction action, String url, String user) { + action.setUrl(url); + action.setUser(user); + action.addHeader("token", DolphinSchedulerTokenManager.getDolphinSchedulerTokenManager(url).getToken(user)); + return getHttpResult(ssoRequestOperation, action); + } + + public static T getHttpGetResult(SSORequestOperation ssoRequestOperation, String url, String user) { + return getHttpResult(ssoRequestOperation, new DSSGetAction(), url, user); + } + + public static T getHttpGetResult(SSORequestOperation ssoRequestOperation, String url, String user, Map parameters) { + DSSGetAction getAction = new DSSGetAction(); + parameters.forEach(getAction::setParameter); + return getHttpResult(ssoRequestOperation, getAction, url, user); + } + + public static T getHttpPostResult(SSORequestOperation ssoRequestOperation, String url, String user, Map formData) { + DSSPostAction postAction = new DSSPostAction(); + formData.forEach(postAction::setParameter); + return getHttpResult(ssoRequestOperation, postAction, url, user); + } + + public static T getHttpPutResult(SSORequestOperation ssoRequestOperation, String url, String user, Map formData) { + DSSPutAction postAction = new DSSPutAction(); + formData.forEach(postAction::addRequestPayload); + return getHttpResult(ssoRequestOperation, postAction, url, user); + } + + public static String getDolphinSchedulerBaseUrl(String baseUrl) { + if(StringUtils.isNotBlank(DolphinSchedulerConf.DOLPHIN_SCHEDULER_URI_PREFIX.getValue())) { + return baseUrl.endsWith("/") ? baseUrl + DolphinSchedulerConf.DOLPHIN_SCHEDULER_URI_PREFIX.getValue() : + baseUrl + "/" + DolphinSchedulerConf.DOLPHIN_SCHEDULER_URI_PREFIX.getValue(); + } else { + return baseUrl; + } + } + + public static long parseToLong(Object val) { + if (val instanceof Double) { + return ((Double) val).longValue(); + } else if (val instanceof Integer) { + return new Double((Integer) val).longValue(); + } else if( val instanceof Long) { + return (Long) val; + } else if(val != null) { + return Long.parseLong(val.toString()); + } + throw new ExternalOperationFailedException(90322, "parse the return of DolphinScheduler failed, the value is null."); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/utils/ProjectUtils.java b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/utils/ProjectUtils.java new file mode 100644 index 000000000..3b7df384b --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/dolphinscheduler/utils/ProjectUtils.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.appconn.dolphinscheduler.utils; + +import org.apache.commons.lang3.StringUtils; + +public class ProjectUtils { + + /** + * 根据DSS空间名和项目名生成DS项目名:DSS空间名-DSS项目名 + * + * @param workspaceName + * @param projectName + * @return + */ + public static String generateDolphinProjectName(String workspaceName, String projectName) { + return StringUtils.joinWith("-", workspaceName, projectName); + } + +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/appconn.properties b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/appconn.properties new file mode 100644 index 000000000..d2a312aa5 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/appconn.properties @@ -0,0 +1,34 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +wds.dss.appconn.ds.admin.user=admin +# please set this property +wds.dss.appconn.ds.admin.token= + +# now, only 1.3.X is supported. +wds.dss.appconn.ds.version=1.3.9 +# this is used for the dolphinscheduler-dss node execution. +wds.dss.appconn.ds.client.home=${DSS_DOLPHINSCHEDULER_CLIENT_HOME} + +# this property is used to add url prefix, if you add a proxy for dolphinscheduler url. +# for example: the normal dolphinscheduler url is http://ip:port/users/create, if you set +# this property, the real url will be http://ip:port/${wds.dss.appconn.ds.url.prefix}/users/create +#wds.dss.appconn.ds.url.prefix= + + + + diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/init.sql b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/init.sql new file mode 100644 index 000000000..07522a339 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/init.sql @@ -0,0 +1,28 @@ +-- 适用于第一次安装时 +select @dolphinscheduler_appconnId:=id from `dss_appconn` where `appconn_name` = 'dolphinscheduler'; +delete from `dss_appconn_instance` where `appconn_id` = @dolphinscheduler_appconnId; + +delete from dss_appconn where appconn_name='dolphinscheduler'; +INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) +VALUES ('dolphinscheduler', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.dss.appconn.dolphinscheduler.DolphinSchedulerAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/dolphinscheduler', ''); + +select @dolphinscheduler_appconnId:=id from `dss_appconn` where `appconn_name` = 'dolphinscheduler'; + +insert into `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`) +values(@dolphinscheduler_appconnId,'DEV','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/','{"reqUri":"dolphinscheduler/projects/analysis/define-user-count"}','dolphinscheduler'); + +-- 看appconn组件是要归属于哪个菜单 +select @dolphinscheduler_menuId:=id from dss_workspace_menu where name = "生产运维"; + +DELETE FROM dss_workspace_menu_appconn where title_en='dolphinscheduler'; +INSERT INTO `dss_workspace_menu_appconn` (`appconn_id`, `menu_id`, `title_en`, `title_cn`, `desc_en`, `desc_cn`, `labels_en`, `labels_cn`, `is_active`, `access_button_en`, `access_button_cn`, `manual_button_en`, `manual_button_cn`, `manual_button_url`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`, `image`) +VALUES(@dolphinscheduler_appconnId, @dolphinscheduler_menuId,'dolphinscheduler','dolphinscheduler','empty desc','empty desc','scheduling, workflow','调度,工作流','1','enter dolphinscheduler','进入dolphinscheduler','user manual','用户手册','manual_url','diaoduxitong-logo',NULL,NULL,NULL,NULL,NULL,'diaoduxitong-icon'); + +delete from dss_workspace_dictionary where dic_key = "pom_work_flow_ds"; +delete from dss_workspace_dictionary where dic_key = "pom_work_flow_ds_DAG"; +insert into `dss_workspace_dictionary`(`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (0,'p_orchestrator_mode','DS工作流','Workflow_DS','pom_work_flow_ds','radio',NULL,NULL,NULL,NULL,0,'gongzuoliu-icon',1,'工程编排模式-DS工作流','SYSTEM','2022-03-21 14:25:35',NULL,'2022-03-21 14:25:35'); +insert into `dss_workspace_dictionary`(`workspace_id`,`parent_key`,`dic_name`,`dic_name_en`,`dic_key`,`dic_value`,`dic_value_en`,`title`,`title_en`,`url`,`url_type`,`icon`,`order_num`,`remark`,`create_user`,`create_time`,`update_user`,`update_time`) values (0,'pom_work_flow_ds','DAG','DAG','pom_work_flow_ds_DAG',NULL,NULL,NULL,NULL,NULL,0,NULL,1,'工程编排模式-DS工作流-DAG','SYSTEM','2022-03-21 14:25:35',NULL,'2022-03-21 14:25:35'); + +INSERT INTO dss_workspace_dictionary +(workspace_id, parent_key, dic_name, dic_name_en, dic_key, dic_value, dic_value_en, title, title_en, url, url_type, icon, order_num, remark, create_user, create_time, update_user, update_time) +VALUES(0, 'p_develop_process', '调度中心', 'Scheduler Center', 'pdp_scheduler_center', 'scheduler', NULL, NULL, NULL, NULL, 0, 'kaifa-icon', 1, '工程开发流程-调度中心', 'SYSTEM', '2020-12-28 17:32:35.0', NULL, '2021-02-22 17:49:02.0'); \ No newline at end of file diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/log4j.properties b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/log4j.properties new file mode 100644 index 000000000..55970acab --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/log4j.properties @@ -0,0 +1,38 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/log4j2.xml b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/log4j2.xml new file mode 100644 index 000000000..6d1dd9a1f --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml b/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml new file mode 100644 index 000000000..0bcd00b2f --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml @@ -0,0 +1,171 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-eventchecker-appconn + + + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-development-process-standard-execution + ${dss.version} + + + + org.apache.commons + commons-lang3 + 3.4 + + + + com.alibaba + druid + 1.0.28 + + + + log4j + log4j + 1.2.17 + + + + org.apache.linkis + linkis-cs-client + ${linkis.version} + + + linkis-common + org.apache.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + + org.apache.linkis + linkis-storage + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.properties + **/application.yml + **/bootstrap.yml + **/log4j2.xml + + + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/assembly/distribution.xml b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..83b2c82f0 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,83 @@ + + + + dss-eventchecker-appconn + + dir + + true + eventchecker + + + + + + lib + true + true + false + true + true + + + + + + ${basedir}/src/main/resources + + appconn.properties + + 0777 + / + unix + + + + ${basedir}/src/main/resources + + log4j.properties + log4j2.xml + + 0777 + conf + unix + + + ${basedir}/src/main/icons + + * + + 0777 + icons + + + + ${basedir}/src/main/resources + + init.sql + + 0777 + db + + + + + + diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/icons/eventreceiver.icon b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/icons/eventreceiver.icon new file mode 100644 index 000000000..facde4d9f --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/icons/eventreceiver.icon @@ -0,0 +1 @@ +eventcheckerCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/icons/eventsender.icon b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/icons/eventsender.icon new file mode 100644 index 000000000..8f2dcea62 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/icons/eventsender.icon @@ -0,0 +1 @@ +eventsenderCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/EventCheckerAppConn.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/EventCheckerAppConn.java new file mode 100644 index 000000000..e0ca3bba1 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/EventCheckerAppConn.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker; + +import com.webank.wedatasphere.dss.appconn.eventchecker.standard.EventCheckerDevelopmentStandard; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.appconn.core.impl.AbstractAppConn; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; + +public class EventCheckerAppConn extends AbstractAppConn implements OnlyDevelopmentAppConn { + + private EventCheckerDevelopmentStandard standard; + + @Override + protected void initialize() { + standard = new EventCheckerDevelopmentStandard(); + } + + @Override + public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() { + return standard; + } + +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/adapter/EventCheckAdapter.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/adapter/EventCheckAdapter.java new file mode 100644 index 000000000..f7004bb38 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/adapter/EventCheckAdapter.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.adapter; + +import org.apache.log4j.Logger; + +import java.util.Properties; + +public interface EventCheckAdapter { + + boolean sendMsg(int jobId, Properties props, Logger log); + + boolean reciveMsg(int jobId, Properties props, Logger log); + +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/connector/EventDruidFactory.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/connector/EventDruidFactory.java new file mode 100644 index 000000000..ca8825fdc --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/connector/EventDruidFactory.java @@ -0,0 +1,118 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.connector; + +import com.alibaba.druid.pool.DruidDataSource; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import java.util.Base64; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + +public class EventDruidFactory { + private static ConcurrentHashMap instanceMap = new ConcurrentHashMap<>(); + private static final String EVENT_DRUID_USERNAME = "msg.eventchecker.jdo.option.username"; + private static final String EVENT_DRUID_URL = "msg.eventchecker.jdo.option.url"; + + public static DruidDataSource getMsgInstance(Properties props, Logger log) { + String eventDruidUsername =props.getProperty(EVENT_DRUID_USERNAME); + String eventDruidUrl = props.getProperty(EVENT_DRUID_URL); + log.info("EVENT_DRUID_USERNAME:" + eventDruidUsername+ ""); + log.info("EVENT_DRUID_URL:" + eventDruidUrl + ""); + String key = eventDruidUsername + eventDruidUrl; + if (instanceMap.containsKey(key)) { + return instanceMap.get(key); + } else { + synchronized (EventDruidFactory.class) { + if (instanceMap.containsKey(key)) { + return instanceMap.get(key); + } + DruidDataSource msgInstance = createDataSource(props, log, "Msg"); + instanceMap.put(key, msgInstance); + return instanceMap.get(key); + } + } + } + + private static DruidDataSource createDataSource(Properties props, Logger log, String type) { + String name = null; + String url = null; + String username = null; + String password = null; + String loginType = null; + + if(type.equals("Msg")){ + name = props.getProperty("msg.eventchecker.jdo.option.name"); + url = props.getProperty("msg.eventchecker.jdo.option.url"); + username = props.getProperty("msg.eventchecker.jdo.option.username"); + loginType = props.getProperty("msg.eventchecker.jdo.option.login.type"); + try { + if("base64".equals(loginType)) { + password = new String(Base64.getDecoder().decode(props.getProperty("msg.eventchecker.jdo.option.password").getBytes()), "UTF-8"); + }else{ + password = props.getProperty("msg.eventchecker.jdo.option.password"); + } + } catch (Exception e){ + log.error("password decore failed" + e); + } + } + + int initialSize = Integer.valueOf(props.getProperty("option.initial.size", "1")); + int maxActive = Integer.valueOf(props.getProperty("option.max.active", "100")); + int minIdle = Integer.valueOf(props.getProperty("option.min.idle", "1")); + long maxWait = Long.valueOf(props.getProperty("option.max.wait", "60000")); + String validationQuery = props.getProperty("option.validation.quert", "SELECT 'x'"); + long timeBetweenEvictionRunsMillis = Long.valueOf(props.getProperty("option.time.between.eviction.runs.millis", "6000")); + long minEvictableIdleTimeMillis = Long.valueOf(props.getProperty("option.evictable.idle,time.millis", "300000")); + boolean testOnBorrow = Boolean.valueOf(props.getProperty("option.test.on.borrow", "true")); + int maxOpenPreparedStatements = Integer.valueOf(props.getProperty("option.max.open.prepared.statements", "-1")); + + if (timeBetweenEvictionRunsMillis > minEvictableIdleTimeMillis) { + timeBetweenEvictionRunsMillis = minEvictableIdleTimeMillis; + } + + DruidDataSource ds = new DruidDataSource(); + + if (StringUtils.isNotBlank(name)) { + ds.setName(name); + } + + ds.setUrl(url); + ds.setDriverClassName("com.mysql.jdbc.Driver"); + ds.setUsername(username); + ds.setPassword(password); + ds.setInitialSize(initialSize); + ds.setMinIdle(minIdle); + ds.setMaxActive(maxActive); + ds.setMaxWait(maxWait); + ds.setTestOnBorrow(testOnBorrow); + ds.setValidationQuery(validationQuery); + ds.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); + ds.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); + if (maxOpenPreparedStatements > 0) { + ds.setPoolPreparedStatements(true); + ds.setMaxPoolPreparedStatementPerConnectionSize( + maxOpenPreparedStatements); + } else { + ds.setPoolPreparedStatements(false); + } + log.info("Druid data source initialed!"); + return ds; + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/cs/CSEventReceiverHelper.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/cs/CSEventReceiverHelper.java new file mode 100644 index 000000000..aa6b65f7f --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/cs/CSEventReceiverHelper.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.cs; + +import org.apache.linkis.cs.client.service.CSVariableService; +import org.apache.linkis.cs.client.utils.ContextServiceUtils; +import org.apache.linkis.cs.client.utils.SerializeHelper; +import org.apache.linkis.cs.common.entity.enumeration.ContextScope; +import org.apache.linkis.cs.common.entity.enumeration.ContextType; +import org.apache.linkis.cs.common.entity.object.LinkisVariable; +import org.apache.linkis.cs.common.entity.source.CommonContextKey; +import org.apache.linkis.cs.common.entity.source.ContextKey; +import org.apache.linkis.cs.common.utils.CSCommonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +public class CSEventReceiverHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(CSEventReceiverHelper.class); + + public static void putVariable(Properties properties, String msgBody, String saveKey) { + String contextIDStr = ContextServiceUtils.getContextIDStrByProperties(properties); + String nodeNameStr = ContextServiceUtils.getNodeNameStrByProperties(properties); + try { + + String key = saveKey; + String value = msgBody; + ContextKey contextKey = new CommonContextKey(); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.OBJECT); + contextKey.setKey(CSCommonUtils.getVariableKey(nodeNameStr, key)); + LinkisVariable varValue = new LinkisVariable(); + varValue.setKey(key); + varValue.setValue(value); + CSVariableService.getInstance().putVariable(contextIDStr, SerializeHelper.serializeContextKey(contextKey), varValue); + } catch (Exception e) { + LOGGER.error("Failed to put variable to cs", e); + } + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java new file mode 100644 index 000000000..d5158d3d1 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java @@ -0,0 +1,227 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.entity; + +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.appconn.eventchecker.service.EventCheckerService; +import com.webank.wedatasphere.dss.appconn.eventchecker.cs.CSEventReceiverHelper; +import com.webank.wedatasphere.dss.appconn.eventchecker.execution.EventCheckerExecutionAction; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class EventChecker implements Runnable{ + public final static String WAIT_TIME = "max.receive.hours"; + public final static String WAIT_FOR_TIME = "wait.for.time"; + public final static String QUERY_FREQUENCY = "query.frequency"; + public final static String MSGTYPE="msg.type"; + public final static String SENDER="msg.sender"; + public final static String RECEIVER="msg.receiver"; + public final static String TOPIC="msg.topic"; + public final static String MSGNAME="msg.name"; + public final static String MSG="msg.body"; + public final static String EXEC_ID = "azkaban.flow.execid"; + public final static String SAVE_KEY="msg.savekey"; + public final static String USER_TIME="msg.init.querytime"; + public final static String TODAY="only.receive.today"; + public final static String AFTERSEND="msg.after.send"; + + private Properties p; + private String jobId; + private int execId; + private EventCheckerService wbDao=null; + private EventCheckerExecutionAction backAction = null; + public Long maxWaitTime; + public int queryFrequency; + + private static Pattern pattern = Pattern.compile("[a-zA-Z_0-9@\\-]+"); + + private static final Logger logger = Logger.getRootLogger(); + + public EventChecker(Properties p, EventCheckerExecutionAction action) { + this.p = p; + this.jobId = "1"; + backAction = action; + String waitTime = p.getProperty(EventChecker.WAIT_TIME, "1"); + Double doubleWaitTime = Double.valueOf(waitTime) * 3600 * 1000; + maxWaitTime = Long.valueOf(doubleWaitTime.longValue()); + String query_frequency = p.getProperty(EventChecker.QUERY_FREQUENCY, "30000"); + queryFrequency = Integer.valueOf(query_frequency); + if(queryFrequency <10000){ + queryFrequency = 10000; + } + } + + @Override + public void run() { + try { + backAction.setState(RefExecutionState.Running); + if (p == null) { + throw new RuntimeException("Properties is null. Can't continue"); + } + if (checkParamMap(p, MSGTYPE)) { + throw new RuntimeException("parameter " + MSGTYPE + " can not be blank."); + } + if (checkParamMap(p, TOPIC)) { + throw new RuntimeException("parameter " + TOPIC + " can not be blank."); + } else { + String topic = p.getProperty(TOPIC); + if (!topic.matches("[^_]*_[^_]*_[^_]*")) { + throw new RuntimeException("Error format of topic parameter. Accept: XX_XX_XX."); + } + } + if (checkParamMap(p, MSGNAME)) { + throw new RuntimeException("parameter " + MSGNAME + " can not be blank."); + } + wbDao = EventCheckerService.getInstance(); + execId = Integer.parseInt(jobId); + boolean success = false; + if (p.getProperty(MSGTYPE).equals("SEND")) { + if (checkParamMap(p, SENDER)) { + throw new RuntimeException("parameter " + SENDER + " can not be blank."); + } else { + String sender = p.getProperty(SENDER); + if (!sender.matches("[^@]*@[^@]*@[^@]*")) { + throw new RuntimeException("Error format of sender parameter. Accept: XX@XX@XX."); + } + } + if (p.containsKey(MSG) && StringUtils.isNotEmpty(p.getProperty(MSG)) && p.getProperty(MSG).length() > 250) { + throw new RuntimeException("parameter " + MSG + " length less than 250 !"); + } + success = wbDao.sendMsg(execId, p, logger); + if (success) { + backAction.setState(RefExecutionState.Success); + + } else { + throw new RuntimeException("Failed Send message."); + } + }else if(p.getProperty(MSGTYPE).equals("RECEIVE")) { + backAction.eventType("RECEIVE"); + receiveMsg(); + } else + { + throw new RuntimeException("Please input correct parameter of msg.type, Select RECEIVE Or SEND."); + } + }catch (Exception ex){ + backAction.setState(RefExecutionState.Failed); + throw ex; + } + + } + + public boolean receiveMsg(){ + boolean success = false; + if(p.getProperty(MSGTYPE).equals("RECEIVE")) { + if (checkParamMap(p, RECEIVER)) { + backAction.setState(RefExecutionState.Failed); + throw new RuntimeException("parameter " + RECEIVER + " can not be blank."); + } else { + String receiver = p.getProperty(RECEIVER); + if (!receiver.matches("[^@]*@[^@]*@[^@]*")) { + backAction.setState(RefExecutionState.Failed); + throw new RuntimeException("Error format of receiver parameter. Accept: XX@XX@XX."); + } + } + String userTime = checkTimeParamMap(p, USER_TIME); + if (StringUtils.isNotEmpty(userTime)) { + p.put(USER_TIME, userTime); + } + success = wbDao.reciveMsg(execId, p, logger); + if (success) { + backAction.saveKeyAndValue(getJobSaveKeyAndValue()); + backAction.setState(RefExecutionState.Success); + } else { + backAction.setState(RefExecutionState.Running); + } + } + return success; + } + + public String getJobSaveKeyAndValue(){ + Map saveValueMap = new HashMap<>(); + String msgBody = p.getProperty(MSG, "{}"); + String saveKey = p.getProperty(SAVE_KEY,"msg.body"); + CSEventReceiverHelper.putVariable(this.p, msgBody, saveKey); + if(StringUtils.isEmpty(saveKey)){ + saveValueMap.put("msg.body", msgBody); + }else { + saveValueMap.put(saveKey, msgBody); + } + Gson gson = new Gson(); + String saveValueJson = gson.toJson(saveValueMap); + logger.info("Output msg body: "+saveValueJson); + return saveValueJson; + } + + public void cancel() throws InterruptedException { + } + + private boolean checkParamMap(Properties p, String key){ + boolean checkFlag = false; + if(!p.containsKey(key)){ + throw new RuntimeException("parameter " + key + " is Empty."); + } + if(p.containsKey(key)){ + if(StringUtils.isEmpty(p.getProperty(key))){ + checkFlag = true; + } + } + if(!MSG.equals(key) && StringUtils.contains(p.getProperty(key), " ")){ + throw new RuntimeException("parameter " + key + " can not contains space !"); + } + if(!checkNoStandardStr(p.getProperty(key))){ + throw new RuntimeException("parameter " + key + " Accept letter and number and _@- only."); + } + if(p.getProperty(key).length() > 45){ + throw new RuntimeException("parameter " + key + " length less than 45 !"); + } + return checkFlag; + } + + private boolean checkNoStandardStr(String param){ + Matcher matcher = pattern.matcher(param); + return matcher.matches(); + } + + private void checkTimeParam(Properties p, String key){ + if(p.containsKey(key)){ + String waitForTime= p.getProperty(key); + if(!waitForTime.matches("^(0?[0-9]|1[0-9]|2[0-3]):(0?[0-9]|[1-5][0-9])$")){ + throw new RuntimeException("Parameter " + key + " Time format error ! For example: HH:mm"); + } + } + } + + private String checkTimeParamMap(Properties p, String key){ + if(p.containsKey(key)){ + String userTime = p.getProperty(key); + Pattern ptime = Pattern.compile("^([1][7-9][0-9][0-9]|[2][0][0-9][0-9])(\\-)([0][1-9]|[1][0-2])(\\-)([0-2][1-9]|[3][0-1])(\\s)([0-1][0-9]|[2][0-3])(:)([0-5][0-9])(:)([0-5][0-9])$"); + Matcher m = ptime.matcher(userTime); + if(!m.matches()){ + throw new RuntimeException("Parameter " + key + " Time format error ! For example: yyyy-MM-dd HH:mm:ss"); + } + return userTime; + }else{ + return null; + } + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/exception/UndefinedPropertyException.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/exception/UndefinedPropertyException.java new file mode 100644 index 000000000..fe86138ea --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/exception/UndefinedPropertyException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.exception; + +/** + * Indicates that a required property is missing from the Props + */ +public class UndefinedPropertyException extends RuntimeException { + + private static final long serialVersionUID = 1; + + public UndefinedPropertyException(final String message) { + super(message); + } + +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java new file mode 100644 index 000000000..e8af842b3 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java @@ -0,0 +1,159 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.service; + +import com.alibaba.druid.pool.DruidDataSource; +import com.webank.wedatasphere.dss.appconn.eventchecker.connector.EventDruidFactory; +import com.webank.wedatasphere.dss.appconn.eventchecker.adapter.EventCheckAdapter; +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.EventChecker; + +import org.apache.log4j.Logger; + +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Enumeration; +import java.util.Properties; + +import javax.sql.DataSource; + +public abstract class AbstractEventCheck implements EventCheckAdapter { + static DataSource msgDS; + String topic; + String msgName; + String receiver; + String sender; + String receiveToday; + String userTime; + String waitTime; + String query_frequency; + String wait_for_time; + String msg; + String afterSend; + + DataSource getMsgDS(Properties props, Logger log) { + msgDS = EventDruidFactory.getMsgInstance(props, log); + if (msgDS == null) { + log.error("Error getting Druid DataSource instance"); + } + return msgDS; + } + + void initECParams(Properties props){ + topic = props.getProperty(EventChecker.TOPIC); + msgName = props.getProperty(EventChecker.MSGNAME); + receiver = props.getProperty(EventChecker.RECEIVER); + sender = props.getProperty(EventChecker.SENDER); + msg = props.getProperty(EventChecker.MSG); + receiveToday = props.getProperty(EventChecker.TODAY); + userTime = props.getProperty(EventChecker.USER_TIME); + waitTime = props.getProperty(EventChecker.WAIT_TIME, "1"); + query_frequency = props.getProperty(EventChecker.QUERY_FREQUENCY, "30000"); + afterSend = props.getProperty(EventChecker.AFTERSEND); + } + + Connection getEventCheckerConnection(Properties props, Logger log){ + Connection connection = null; + try { + connection = getMsgDS(props,log).getConnection(); + } catch (SQLException e) { + throw new RuntimeException("Error getting DB Connection instance {} " + e); + } + return connection; + } + + @Override + public boolean sendMsg(int jobId, Properties props, Logger log) { + return false; + } + + @Override + public boolean reciveMsg(int jobId, Properties props, Logger log) { + return false; + } + + void closeConnection(Connection conn, Logger log) { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + log.error("Error closing connection", e); + } + } + } + + void closeQueryRef(ResultSet rs, Logger log) { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + log.error("Error closing result set", e); + } + } + + } + + void closeQueryStmt(PreparedStatement stmt, Logger log) { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + log.error("Error closing result stmt", e); + } + } + + } + + + public static void closeDruidDataSource() { + DruidDataSource msgDSObject = (DruidDataSource) msgDS; + if (msgDSObject != null) { + msgDSObject.close(); + } + + } + + String getLinuxLocalIp(Logger log) { + String ip = "127.0.0.1"; + try { + for (Enumeration en = NetworkInterface.getNetworkInterfaces(); en.hasMoreElements(); ) { + NetworkInterface intf = en.nextElement(); + String name = intf.getName(); + if (!name.contains("docker") && !name.contains("lo")) { + for (Enumeration enumIpAddr = intf.getInetAddresses(); enumIpAddr.hasMoreElements(); ) { + InetAddress inetAddress = enumIpAddr.nextElement(); + if (!inetAddress.isLoopbackAddress()) { + String ipaddress = inetAddress.getHostAddress().toString(); + if (!ipaddress.contains("::") && !ipaddress.contains("0:0:") && !ipaddress.contains("fe80")) { + ip = ipaddress; + } + } + } + } + } + } catch (SocketException ex) { + log.warn("get ip failed", ex); + + } + log.info("Send IP:" + ip); + return ip; + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheckReceiver.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheckReceiver.java new file mode 100644 index 000000000..dd48920fd --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheckReceiver.java @@ -0,0 +1,176 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.service; + +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.EventChecker; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.log4j.Logger; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; +import java.util.Properties; + +public class AbstractEventCheckReceiver extends AbstractEventCheck{ + /** + * Fill the result into the source + */ + String setConsumedMsg(Properties props, Logger log, String[] consumedMsgInfo){ + String vNewMsgID = ""; + try { + if(consumedMsgInfo!=null && consumedMsgInfo.length == 4){ + vNewMsgID = consumedMsgInfo[0]; + String vMsgName = consumedMsgInfo[1]; + String vSender = consumedMsgInfo[2]; + String vMsg = consumedMsgInfo[3]; + if (null == vMsg) { + props.put(EventChecker.MSG, "NULL"); + } else { + props.put(EventChecker.MSG, vMsg); + } + log.info("Received message : messageID: " + vNewMsgID + ", messageName: " + vMsgName + ", receiver: " + vSender + + ", messageBody: " + vMsg); + } + }catch (Exception e) { + log.error("Error set consumed message failed {} setConsumedMsg failed" + e); + return vNewMsgID; + } + return vNewMsgID; + } + + /** + * Update consumption status + */ + boolean updateMsgOffset(int jobId, Properties props, Logger log, String[] consumedMsgInfo,String lastMsgId){ + boolean result = false; + String vNewMsgID = "-1"; + PreparedStatement updatePstmt = null; + Connection msgConn = null; + vNewMsgID = setConsumedMsg(props,log,consumedMsgInfo); + try { + if(StringUtils.isNotEmpty(vNewMsgID) && StringUtils.isNotBlank(vNewMsgID) && !"-1".equals(vNewMsgID)){ + msgConn = getEventCheckerConnection(props,log); + if(msgConn == null) return false; + int vProcessID = jobId; + String vReceiveTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss");; + String sqlForUpdateMsg = "INSERT INTO event_status(receiver,topic,msg_name,receive_time,msg_id) VALUES(?,?,?,?,?) ON DUPLICATE KEY UPDATE receive_time=VALUES(receive_time),msg_id= CASE WHEN msg_id= " + lastMsgId + " THEN VALUES(msg_id) ELSE msg_id END"; + log.info("last message offset {} is:" + lastMsgId); + updatePstmt = msgConn.prepareCall(sqlForUpdateMsg); + updatePstmt.setString(1, receiver); + updatePstmt.setString(2, topic); + updatePstmt.setString(3, msgName); + updatePstmt.setString(4, vReceiveTime); + updatePstmt.setString(5, vNewMsgID); + int updaters = updatePstmt.executeUpdate(); + log.info("updateMsgOffset successful {} update result is:" + updaters); + if(updaters != 0){ + log.info("Received message successfully , update message status succeeded, consumed flow execution ID: " + vProcessID); + //return true after update success + result = true; + }else{ + log.info("Received message successfully , update message status failed, consumed flow execution ID: " + vProcessID); + result = false; + } + }else{ + result = false; + } + }catch (SQLException e){ + log.error("Error update Msg Offset" + e); + return false; + }finally { + closeQueryStmt(updatePstmt, log); + closeConnection(msgConn, log); + } + return result; + } + + /** + * get consumption progress + */ + String getOffset(int jobId, Properties props, Logger log){ + String sqlForReadMsgID = "SELECT msg_id FROM event_status WHERE receiver=? AND topic=? AND msg_name=?"; + PreparedStatement pstmtForGetID = null; + Connection msgConn = null; + ResultSet rs = null; + boolean flag = false; + String lastMsgId = "0"; + try { + msgConn = getEventCheckerConnection(props,log); + pstmtForGetID = msgConn.prepareCall(sqlForReadMsgID); + pstmtForGetID.setString(1, receiver); + pstmtForGetID.setString(2, topic); + pstmtForGetID.setString(3, msgName); + rs = pstmtForGetID.executeQuery(); + lastMsgId = rs.last()==true ? rs.getString("msg_id"):"0"; + } catch (SQLException e) { + throw new RuntimeException("get Offset failed " + e); + }finally { + closeQueryStmt(pstmtForGetID,log); + closeConnection(msgConn,log); + closeQueryRef(rs,log); + } + log.info("The last record id was " + lastMsgId); + return lastMsgId; + } + + /** + * Consistent entrance to consumer message + */ + String[] getMsg(Properties props, Logger log,String ... params){ + String sqlForReadTMsg = "SELECT * FROM event_queue WHERE topic=? AND msg_name=? AND send_time >=? AND send_time <=? AND msg_id >? ORDER BY msg_id ASC LIMIT 1"; + PreparedStatement pstmt = null; + Connection msgConn = null; + ResultSet rs = null; + String[] consumedMsgInfo = null; + try { + msgConn = getEventCheckerConnection(props,log); + pstmt = msgConn.prepareCall(sqlForReadTMsg); + pstmt.setString(1, topic); + pstmt.setString(2, msgName); + pstmt.setString(3, params[0]); + pstmt.setString(4, params[1]); + pstmt.setString(5, params[2]); + log.info("param {} StartTime: " + params[0] + ", EndTime: " + params[1] + + ", Topic: " + topic + ", MessageName: " + msgName + ", LastMessageID: " + params[2]); + rs = pstmt.executeQuery(); + + if(rs.last()){ + consumedMsgInfo = new String[4]; + String[] msgKey = new String[]{"msg_id","msg_name","sender","msg"}; + for (int i = 0;i <= 3;i++) { + consumedMsgInfo[i] = rs.getString(msgKey[i]); + } + } + } catch (SQLException e) { + throw new RuntimeException("EventChecker failed to receive message" + e); + } finally { + closeQueryStmt(pstmt, log); + closeConnection(msgConn, log); + closeQueryRef(rs, log); + } + return consumedMsgInfo; + } + + @Override + public boolean reciveMsg(int jobId, Properties props, Logger log) { + return super.reciveMsg(jobId, props, log); + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/DefaultEventcheckReceiver.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/DefaultEventcheckReceiver.java new file mode 100644 index 000000000..9fa17274f --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/DefaultEventcheckReceiver.java @@ -0,0 +1,142 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.service; + + + +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.log4j.Logger; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Properties; + +public class DefaultEventcheckReceiver extends AbstractEventCheckReceiver { + String todayStartTime; + String todayEndTime; + String allStartTime; + String allEndTime; + String nowStartTime; + + public DefaultEventcheckReceiver(Properties props) { + initECParams(props); + initReceiverTimes(); + } + + private void initReceiverTimes(){ + todayStartTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd 00:00:00"); + todayEndTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd 23:59:59"); + allStartTime = DateFormatUtils.format(new Date(), "10000-01-01 00:00:00"); + allEndTime = DateFormatUtils.format(new Date(), "9999-12-31 23:59:59"); + nowStartTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss"); + } + + @Override + public boolean reciveMsg(int jobId, Properties props, Logger log) { + boolean result = false; + try{ + String lastMsgId = getOffset(jobId,props,log); + String[] executeType = createExecuteType(jobId,props,log,lastMsgId); + if(executeType!=null && executeType.length ==3){ + String[] consumedMsgInfo = getMsg(props, log,executeType); + if(consumedMsgInfo!=null && consumedMsgInfo.length == 4){ + result = updateMsgOffset(jobId,props,log,consumedMsgInfo,lastMsgId); + } + }else{ + log.error("executeType error {} " + executeType.toString()); + return result; + } + }catch (Exception e){ + log.error("EventChecker failed to receive the message {}" + e); + return result; + } + return result; + } + + private String[] createExecuteType(int jobId, Properties props, Logger log,String lastMsgId){ + boolean receiveTodayFlag = (null != receiveToday && "true".equals(receiveToday.trim().toLowerCase())); + boolean afterSendFlag = (null != afterSend && "true".equals(afterSend.trim().toLowerCase())); + String[] executeType = null; + try { + if ("0".equals(lastMsgId)){ + if(receiveTodayFlag){ + if(afterSendFlag){ + executeType = new String[]{nowStartTime,todayEndTime,"0"}; + }else{ + executeType = new String[]{todayStartTime,todayEndTime,"0"}; + } + }else{ + if(afterSendFlag){ + executeType = new String[]{nowStartTime,allEndTime,"0"}; + }else{ + executeType = new String[]{allStartTime,allEndTime,"0"}; + } + } + }else{ + if(receiveTodayFlag){ + if(afterSendFlag){ + executeType = new String[]{nowStartTime,todayEndTime,lastMsgId}; + }else{ + executeType = new String[]{todayStartTime,todayEndTime,lastMsgId}; + } + }else{ + if(afterSendFlag){ + executeType = new String[]{nowStartTime,allEndTime,lastMsgId}; + }else{ + executeType = new String[]{allStartTime,allEndTime,lastMsgId}; + } + } + } + }catch(Exception e){ + log.error("create executeType failed {}" + e); + } + return executeType; + } + + private void waitForTime(Logger log,Long waitTime){ + String waitForTime = wait_for_time; + String formatWaitForTime = DateFormatUtils.format(new Date(),"yyyy-MM-dd " + waitForTime + ":00"); + DateFormat fmt =new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date targetWaitTime = null; + try { + targetWaitTime = fmt.parse(formatWaitForTime); + } catch (ParseException e) { + log.error("parse date failed {}" + e); + } + + log.info("It will success at a specified time: " + targetWaitTime); + long wt = targetWaitTime.getTime() - System.currentTimeMillis(); + if(wt > 0){ + //wt must less than wait.time + if(wt <= waitTime){ + log.info("EventChecker will wait "+ wt + " milliseconds before starting execution"); + try { + Thread.sleep(wt); + } catch (InterruptedException e) { + throw new RuntimeException("EventChecker throws an exception during the waiting time {}"+e); + } + }else{ + throw new RuntimeException("The waiting time from Job starttime to wait.for.time"+ wt +"(ms) greater than wait.time , unreasonable setting!"); + } + }else{ + log.info("EventChecker has reached the specified time"); + } + } + +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckSender.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckSender.java new file mode 100644 index 000000000..8e21ab72d --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckSender.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.service; + +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.log4j.Logger; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Date; +import java.util.Properties; + +public class EventCheckSender extends AbstractEventCheck { + + public EventCheckSender(Properties props) { + initECParams(props); + } + + @Override + public boolean sendMsg(int jobId, Properties props, Logger log) { + boolean result = false; + PreparedStatement pstmt = null; + Connection msgConn = null; + String sendTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss"); + String sqlForSendMsg = "INSERT INTO event_queue (sender,send_time,topic,msg_name,msg,send_ip) VALUES(?,?,?,?,?,?)"; + try { + String vIP = getLinuxLocalIp(log); + msgConn = getEventCheckerConnection(props,log); + if(msgConn==null) return false; + pstmt = msgConn.prepareCall(sqlForSendMsg); + pstmt.setString(1, sender); + pstmt.setString(2, sendTime); + pstmt.setString(3, topic); + pstmt.setString(4, msgName); + pstmt.setString(5, msg); + pstmt.setString(6, vIP); + int rs = pstmt.executeUpdate(); + if (rs == 1) { + result = true; + log.info("Send msg success!"); + } else { + log.error("Send msg failed for update database!"); + } + } catch (SQLException e) { + throw new RuntimeException("Send EventChecker msg failed!" + e); + } finally { + closeQueryStmt(pstmt, log); + closeConnection(msgConn, log); + } + return result; + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerExecuteService.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerExecuteService.java new file mode 100644 index 000000000..115a81ee1 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerExecuteService.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.service; + +import com.webank.wedatasphere.dss.appconn.eventchecker.execution.EventCheckerRefExecutionOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExecutionService; + +public class EventCheckerExecuteService extends AbstractRefExecutionService { + + @Override + protected EventCheckerRefExecutionOperation createRefExecutionOperation() { + return new EventCheckerRefExecutionOperation(); + } + +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerService.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerService.java new file mode 100644 index 000000000..2ca27bec3 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerService.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.service; + +import org.apache.log4j.Logger; + +import java.util.Properties; + +public class EventCheckerService { + private static EventCheckerService instance; + + public static EventCheckerService getInstance() { + if (instance == null) { + synchronized (EventCheckerService.class) { + if (instance == null) { + instance = new EventCheckerService(); + } + } + } + return instance; + } + + public boolean sendMsg(int jobId, Properties props, Logger log) { + if(props!=null){ + return new EventCheckSender(props).sendMsg(jobId,props,log); + }else{ + log.error("create EventCheckSender failed {}"); + return false; + } + } + + /** + * Receiving a message first queries the consumption record, + * and then starts to consume after the last consumption, and no consumption + * starts after the job starts. The received message is performed in an active + * query manner, and the target message is repeatedly queried within a time period + * when the set target is not exceeded. + */ + public boolean reciveMsg(int jobId, Properties props, Logger log) { + if(props!=null){ + return new DefaultEventcheckReceiver(props).reciveMsg(jobId,props,log); + }else{ + log.error("create EventCheckSender failed {}"); + return false; + } + } + +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/standard/EventCheckerDevelopmentStandard.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/standard/EventCheckerDevelopmentStandard.java new file mode 100644 index 000000000..fa8dfb0b2 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/standard/EventCheckerDevelopmentStandard.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.standard; + +import com.webank.wedatasphere.dss.appconn.eventchecker.service.EventCheckerExecuteService; +import com.webank.wedatasphere.dss.standard.app.development.service.RefExecutionService; +import com.webank.wedatasphere.dss.standard.app.development.standard.OnlyExecutionDevelopmentStandard; + +public class EventCheckerDevelopmentStandard extends OnlyExecutionDevelopmentStandard { + + @Override + protected RefExecutionService createRefExecutionService() { + return new EventCheckerExecuteService(); + } + + + @Override + public void init() {} + + + @Override + public String getStandardName() { + return "EventCheckDevelopmentStandard"; + } + +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/Props.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/Props.java new file mode 100644 index 000000000..e4e8cc9b8 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/Props.java @@ -0,0 +1,850 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.utils; + +import com.webank.wedatasphere.dss.appconn.eventchecker.exception.UndefinedPropertyException; + +import org.apache.log4j.Logger; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; + +/** + * Hashmap implementation of a hierarchical properties with helpful converter functions and + * Exception throwing. This class is not threadsafe. + */ +public class Props { + + private final Map _current; + private Props _parent; + private String source = null; + + /** + * Constructor for empty props with empty parent. + */ + public Props() { + this(null); + } + + /** + * Constructor for empty Props with parent override. + */ + public Props(final Props parent) { + this._current = new HashMap(); + this._parent = parent; + } + + /** + * Load props from a file. + */ + public Props(final Props parent, final String filepath) throws IOException { + this(parent, new File(filepath)); + } + + /** + * Load props from a file. + */ + public Props(final Props parent, final File file) throws IOException { + this(parent); + setSource(file.getPath()); + + final InputStream input = new BufferedInputStream(new FileInputStream(file)); + try { + loadFrom(input); + } catch (final IOException e) { + throw e; + } finally { + input.close(); + } + } + + /** + * Create props from property input streams + */ + public Props(final Props parent, final InputStream inputStream) throws IOException { + this(parent); + loadFrom(inputStream); + } + + /** + * Create properties from maps of properties + */ + public Props(final Props parent, final Map... props) { + this(parent); + for (int i = props.length - 1; i >= 0; i--) { + this.putAll(props[i]); + } + } + + /** + * Create properties from Properties objects + */ + public Props(final Props parent, final Properties... properties) { + this(parent); + for (int i = properties.length - 1; i >= 0; i--) { + this.put(properties[i]); + } + } + + /** + * Create a Props object with the contents set to that of props. + */ + public Props(final Props parent, final Props props) { + this(parent); + if (props != null) { + putAll(props); + } + } + + /** + * Create a Props with a null parent from a list of key value pairing. i.e. [key1, value1, key2, + * value2 ...] + */ + public static Props of(final String... args) { + return of((Props) null, args); + } + + /** + * Create a Props from a list of key value pairing. i.e. [key1, value1, key2, value2 ...] + */ + public static Props of(final Props parent, final String... args) { + if (args.length % 2 != 0) { + throw new IllegalArgumentException( + "Must have an equal number of keys and values."); + } + + final Map vals = new HashMap<>(args.length / 2); + + for (int i = 0; i < args.length; i += 2) { + vals.put(args[i], args[i + 1]); + } + return new Props(parent, vals); + } + + /** + * Clones the Props p object and all of its parents. + */ + public static Props clone(final Props p) { + return copyNext(p); + } + + private static Props copyNext(final Props source) { + Props priorNodeCopy = null; + if (source.getParent() != null) { + priorNodeCopy = copyNext(source.getParent()); + } + final Props dest = new Props(priorNodeCopy); + for (final String key : source.localKeySet()) { + dest.put(key, source.get(key)); + } + + return dest; + } + + private void loadFrom(final InputStream inputStream) throws IOException { + final Properties properties = new Properties(); + // Solve the problem that the. Job file contains Chinese and reads garbled code. + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream , "UTF-8")); + properties.load(bufferedReader); + this.put(properties); + } + + public Props getEarliestAncestor() { + if (this._parent == null) { + return this; + } + + return this._parent.getEarliestAncestor(); + } + + public void setEarliestAncestor(final Props parent) { + final Props props = getEarliestAncestor(); + props.setParent(parent); + } + + /** + * Clear the current Props, but leaves the parent untouched. + */ + public void clearLocal() { + this._current.clear(); + } + + /** + * Check key in current Props then search in parent + */ + public boolean containsKey(final Object k) { + return this._current.containsKey(k) + || (this._parent != null && this._parent.containsKey(k)); + } + + /** + * Check value in current Props then search in parent + */ + public boolean containsValue(final Object value) { + return this._current.containsValue(value) + || (this._parent != null && this._parent.containsValue(value)); + } + + /** + * Return value if available in current Props otherwise return from parent + */ + public String get(final Object key) { + if (this._current.containsKey(key)) { + return this._current.get(key); + } else if (this._parent != null) { + return this._parent.get(key); + } else { + return null; + } + } + + /** + * Get the key set from the current Props + */ + public Set localKeySet() { + return this._current.keySet(); + } + + /** + * Get parent Props + */ + public Props getParent() { + return this._parent; + } + + public void setParent(final Props prop) { + this._parent = prop; + } + + /** + * Put the given string value for the string key. This method performs any variable substitution + * in the value replacing any occurance of ${name} with the value of get("name"). + * + * @param key The key to put the value to + * @param value The value to do substitution on and store + * @throws IllegalArgumentException If the variable given for substitution is not a valid key in + * this Props. + */ + public String put(final String key, final String value) { + return this._current.put(key, value); + } + + /** + * Put the given Properties into the Props. This method performs any variable substitution in the + * value replacing any occurrence of ${name} with the value of get("name"). get() is called first + * on the Props and next on the Properties object. + * + * @param properties The properties to put + * @throws IllegalArgumentException If the variable given for substitution is not a valid key in + * this Props. + */ + public void put(final Properties properties) { + for (final String propName : properties.stringPropertyNames()) { + this._current.put(propName, properties.getProperty(propName)); + } + } + + /** + * Put integer + */ + public String put(final String key, final Integer value) { + return this._current.put(key, value.toString()); + } + + /** + * Put Long. Stores as String. + */ + public String put(final String key, final Long value) { + return this._current.put(key, value.toString()); + } + + /** + * Put Double. Stores as String. + */ + public String put(final String key, final Double value) { + return this._current.put(key, value.toString()); + } + + /** + * Put everything in the map into the props. + */ + public void putAll(final Map m) { + if (m == null) { + return; + } + + for (final Map.Entry entry : m.entrySet()) { + this.put(entry.getKey(), entry.getValue()); + } + } + + /** + * Put all properties in the props into the current props. Will handle null p. + */ + public void putAll(final Props p) { + if (p == null) { + return; + } + + for (final String key : p.getKeySet()) { + this.put(key, p.get(key)); + } + } + + /** + * Puts only the local props from p into the current properties + */ + public void putLocal(final Props p) { + for (final String key : p.localKeySet()) { + this.put(key, p.get(key)); + } + } + + /** + * Remove only the local value of key s, and not the parents. + */ + public String removeLocal(final Object s) { + return this._current.remove(s); + } + + /** + * The number of unique keys defined by this Props and all parent Props + */ + public int size() { + return getKeySet().size(); + } + + /** + * The number of unique keys defined by this Props (keys defined only in parent Props are not + * counted) + */ + public int localSize() { + return this._current.size(); + } + + /** + * Attempts to return the Class that corresponds to the Props value. If the class doesn't exit, an + * IllegalArgumentException will be thrown. + */ + public Class getClass(final String key) { + try { + if (containsKey(key)) { + return Class.forName(get(key)); + } else { + throw new UndefinedPropertyException("Missing required property '" + + key + "'"); + } + } catch (final ClassNotFoundException e) { + throw new IllegalArgumentException(e); + } + } + + public Class getClass(final String key, final boolean initialize, final ClassLoader cl) { + try { + if (containsKey(key)) { + return Class.forName(get(key), initialize, cl); + } else { + throw new UndefinedPropertyException("Missing required property '" + + key + "'"); + } + } catch (final ClassNotFoundException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Gets the class from the Props. If it doesn't exist, it will return the defaultClass + */ + public Class getClass(final String key, final Class defaultClass) { + if (containsKey(key)) { + return getClass(key); + } else { + return defaultClass; + } + } + + /** + * Gets the string from the Props. If it doesn't exist, it will return the defaultValue + */ + public String getString(final String key, final String defaultValue) { + if (containsKey(key)) { + return get(key); + } else { + return defaultValue; + } + } + + /** + * Gets the string from the Props. If it doesn't exist, throw and UndefinedPropertiesException + */ + public String getString(final String key) { + if (containsKey(key)) { + return get(key); + } else { + throw new UndefinedPropertyException("Missing required property '" + key + + "'"); + } + } + + /** + * Returns a list of strings with the comma as the separator of the value + */ + public List getStringList(final String key) { + return getStringList(key, "\\s*,\\s*"); + } + + /** + * Returns a list of clusters with the comma as the separator of the value + * e.g., for input string: "thrift://hcat1:port,thrift://hcat2:port;thrift://hcat3:port,thrift://hcat4:port;" + * we will get ["thrift://hcat1:port,thrift://hcat2:port", "thrift://hcat3:port,thrift://hcat4:port"] as output + */ + public List getStringListFromCluster(final String key) { + List curlist = getStringList(key, "\\s*;\\s*"); + // remove empty elements in the array + for (Iterator iter = curlist.listIterator(); iter.hasNext(); ) { + String a = iter.next(); + if (a.length() == 0) { + iter.remove(); + } + } + return curlist; + } + + /** + * Returns a list of strings with the sep as the separator of the value + */ + public List getStringList(final String key, final String sep) { + final String val = get(key); + if (val == null || val.trim().length() == 0) { + return Collections.emptyList(); + } + + if (containsKey(key)) { + return Arrays.asList(val.split(sep)); + } else { + throw new UndefinedPropertyException("Missing required property '" + key + + "'"); + } + } + + /** + * Returns a list of strings with the comma as the separator of the value. If the value is null, + * it'll return the defaultValue. + */ + public List getStringList(final String key, final List defaultValue) { + if (containsKey(key)) { + return getStringList(key); + } else { + return defaultValue; + } + } + + /** + * Returns a list of strings with the sep as the separator of the value. If the value is null, + * it'll return the defaultValue. + */ + public List getStringList(final String key, final List defaultValue, + final String sep) { + if (containsKey(key)) { + return getStringList(key, sep); + } else { + return defaultValue; + } + } + + /** + * Returns true if the value equals "true". If the value is null, then the default value is + * returned. + */ + public boolean getBoolean(final String key, final boolean defaultValue) { + if (containsKey(key)) { + return "true".equalsIgnoreCase(get(key).trim()); + } else { + return defaultValue; + } + } + + /** + * Returns true if the value equals "true". If the value is null, then an + * UndefinedPropertyException is thrown. + */ + public boolean getBoolean(final String key) { + if (containsKey(key)) { + return "true".equalsIgnoreCase(get(key)); + } else { + throw new UndefinedPropertyException("Missing required property '" + key + + "'"); + } + } + + /** + * Returns the long representation of the value. If the value is null, then the default value is + * returned. If the value isn't a long, then a parse exception will be thrown. + */ + public long getLong(final String name, final long defaultValue) { + if (containsKey(name)) { + return Long.parseLong(get(name)); + } else { + return defaultValue; + } + } + + /** + * Returns the long representation of the value. If the value is null, then a + * UndefinedPropertyException will be thrown. If the value isn't a long, then a parse exception + * will be thrown. + */ + public long getLong(final String name) { + if (containsKey(name)) { + return Long.parseLong(get(name)); + } else { + throw new UndefinedPropertyException("Missing required property '" + name + + "'"); + } + } + + /** + * Returns the int representation of the value. If the value is null, then the default value is + * returned. If the value isn't a int, then a parse exception will be thrown. + */ + public int getInt(final String name, final int defaultValue) { + if (containsKey(name)) { + return Integer.parseInt(get(name).trim()); + } else { + return defaultValue; + } + } + + /** + * Returns the int representation of the value. If the value is null, then a + * UndefinedPropertyException will be thrown. If the value isn't a int, then a parse exception + * will be thrown. + */ + public int getInt(final String name) { + if (containsKey(name)) { + return Integer.parseInt(get(name).trim()); + } else { + throw new UndefinedPropertyException("Missing required property '" + name + + "'"); + } + } + + /** + * Returns the double representation of the value. If the value is null, then the default value is + * returned. If the value isn't a double, then a parse exception will be thrown. + */ + public double getDouble(final String name, final double defaultValue) { + if (containsKey(name)) { + return Double.parseDouble(get(name).trim()); + } else { + return defaultValue; + } + } + + /** + * Returns the double representation of the value. If the value is null, then a + * UndefinedPropertyException will be thrown. If the value isn't a double, then a parse exception + * will be thrown. + */ + public double getDouble(final String name) { + if (containsKey(name)) { + return Double.parseDouble(get(name).trim()); + } else { + throw new UndefinedPropertyException("Missing required property '" + name + + "'"); + } + } + + /** + * Returns the uri representation of the value. If the value is null, then the default value is + * returned. If the value isn't a uri, then a IllegalArgumentException will be thrown. + */ + public URI getUri(final String name) { + if (containsKey(name)) { + try { + return new URI(get(name)); + } catch (final URISyntaxException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } else { + throw new UndefinedPropertyException("Missing required property '" + name + + "'"); + } + } + + /** + * Returns the double representation of the value. If the value is null, then the default value is + * returned. If the value isn't a uri, then a IllegalArgumentException will be thrown. + */ + public URI getUri(final String name, final URI defaultValue) { + if (containsKey(name)) { + return getUri(name); + } else { + return defaultValue; + } + } + + public URI getUri(final String name, final String defaultValue) { + try { + return getUri(name, new URI(defaultValue)); + } catch (final URISyntaxException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } + + /** + * Store only those properties defined at this local level + * + * @param file The file to write to + * @throws IOException If the file can't be found or there is an io error + */ + public void storeLocal(final File file) throws IOException { + final BufferedOutputStream out = + new BufferedOutputStream(new FileOutputStream(file)); + try { + storeLocal(out); + } finally { + out.close(); + } + } + + /** + * Returns a copy of only the local values of this props + */ + public Props local() { + return new Props(null, this._current); + } + + /** + * Store only those properties defined at this local level + * + * @param out The output stream to write to + * @throws IOException If the file can't be found or there is an io error + */ + public void storeLocal(final OutputStream out) throws IOException { + final Properties p = new Properties(); + for (final String key : this._current.keySet()) { + p.setProperty(key, get(key)); + } + p.store(out, null); + } + + /** + * Returns a java.util.Properties file populated with the current Properties in here. + * Note: if you want to import parent properties (e.g., database credentials), please use + * toAllProperties + */ + public Properties toProperties() { + final Properties p = new Properties(); + for (final String key : this._current.keySet()) { + p.setProperty(key, get(key)); + } + + return p; + } + + /** + * Returns a java.util.Properties file populated with both current and parent properties. + */ + public Properties toAllProperties() { + Properties allProp = new Properties(); + // import local properties + allProp.putAll(toProperties()); + // import parent properties + if(_parent != null){ + allProp.putAll(_parent.toProperties()); + } + return allProp; + } + + /** + * Store all properties, those local and also those in parent props + * + * @param file The file to store to + * @throws IOException If there is an error writing + */ + public void storeFlattened(final File file) throws IOException { + final BufferedOutputStream out = + new BufferedOutputStream(new FileOutputStream(file)); + try { + storeFlattened(out); + } finally { + out.close(); + } + } + + /** + * Store all properties, those local and also those in parent props + * + * @param out The stream to write to + * @throws IOException If there is an error writing + */ + public void storeFlattened(final OutputStream out) throws IOException { + final Properties p = new Properties(); + for (Props curr = this; curr != null; curr = curr.getParent()) { + for (final String key : curr.localKeySet()) { + if (!p.containsKey(key)) { + p.setProperty(key, get(key)); + } + } + } + + p.store(out, null); + } + + /** + * Returns a map of all the flattened properties, the item in the returned map is sorted + * alphabetically by the key value. + * + * @Return + */ + public Map getFlattened() { + final TreeMap returnVal = new TreeMap<>(); + returnVal.putAll(getMapByPrefix("")); + return returnVal; + } + + /** + * Get a map of all properties by string prefix + * + * @param prefix The string prefix + */ + public Map getMapByPrefix(final String prefix) { + final Map values = this._parent == null ? new HashMap<>() : + this._parent.getMapByPrefix(prefix); + + // when there is a conflict, value from the child takes the priority. + for (final String key : this.localKeySet()) { + if (key.startsWith(prefix)) { + values.put(key.substring(prefix.length()), get(key)); + } + } + return values; + } + + /** + * Returns a set of all keys, including the parents + */ + public Set getKeySet() { + final HashSet keySet = new HashSet<>(); + + keySet.addAll(localKeySet()); + + if (this._parent != null) { + keySet.addAll(this._parent.getKeySet()); + } + + return keySet; + } + + /** + * Logs the property in the given logger + */ + public void logProperties(final Logger logger, final String comment) { + logger.info(comment); + + for (final String key : getKeySet()) { + logger.info(" key=" + key + " value=" + get(key)); + } + } + + @Override + public boolean equals(final Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o.getClass() != Props.class) { + return false; + } + + final Props p = (Props) o; + return this._current.equals(p._current) && Utils.equals(this._parent, p._parent); + } + + /** + * Returns true if the properties are equivalent, regardless of the hierarchy. + */ + public boolean equalsProps(final Props p) { + if (p == null) { + return false; + } + + final Set myKeySet = getKeySet(); + for (final String s : myKeySet) { + if (!get(s).equals(p.get(s))) { + return false; + } + } + + return myKeySet.size() == p.getKeySet().size(); + } + + @Override + public int hashCode() { + int code = this._current.hashCode(); + if (this._parent != null) { + code += this._parent.hashCode(); + } + return code; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder("{"); + for (final Map.Entry entry : this._current.entrySet()) { + builder.append(entry.getKey()); + builder.append(": "); + builder.append(entry.getValue()); + builder.append(", "); + } + if (this._parent != null) { + builder.append(" parent = "); + builder.append(this._parent.toString()); + } + builder.append("}"); + return builder.toString(); + } + + public String getSource() { + return this.source; + } + + public void setSource(final String source) { + this.source = source; + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/Utils.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/Utils.java new file mode 100644 index 000000000..a53dbf27e --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/Utils.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.utils; + +/** + * A util helper class full of static methods that are commonly used. + */ +public class Utils { + /** + * Equivalent to Object.equals except that it handles nulls. If a and b are both null, true is + * returned. + */ + public static boolean equals(final Object a, final Object b) { + if (a == null || b == null) { + return a == b; + } + + return a.equals(b); + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/appjoint.properties b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/appjoint.properties new file mode 100644 index 000000000..9538027d4 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/appjoint.properties @@ -0,0 +1,24 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +msg.eventchecker.jdo.option.name=msg +msg.eventchecker.jdo.option.url=jdbc:mysql://127.0..0.1:3306/ +msg.eventchecker.jdo.option.username=user +msg.eventchecker.jdo.option.password= +msg.eventchecker.jdo.option.login.type=base64 + + + diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/init.sql b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/init.sql new file mode 100644 index 000000000..d35ef6a3d --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/init.sql @@ -0,0 +1,98 @@ +DROP TABLE IF EXISTS `event_queue`; +CREATE TABLE `event_queue` ( + `msg_id` int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT '消息ID号', + `sender` varchar(45) NOT NULL COMMENT '消息发送者', + `send_time` datetime NOT NULL COMMENT '消息发送时间', + `topic` varchar(45) NOT NULL COMMENT '消息主题', + `msg_name` varchar(45) NOT NULL COMMENT '消息名称', + `msg` varchar(250) DEFAULT NULL COMMENT '消息内容', + `send_ip` varchar(45) NOT NULL, + PRIMARY KEY (`msg_id`) +) ENGINE=InnoDB AUTO_INCREMENT=21068 DEFAULT CHARSET=utf8 COMMENT='azkaban调取系统消息队列表'; + +DROP TABLE IF EXISTS `event_status`; +CREATE TABLE `event_status` ( + `receiver` varchar(45) NOT NULL COMMENT '消息接收者', + `receive_time` datetime NOT NULL COMMENT '消息接收时间', + `topic` varchar(45) NOT NULL COMMENT '消息主题', + `msg_name` varchar(45) NOT NULL COMMENT '消息名称', + `msg_id` int(11) NOT NULL COMMENT '消息的最大消费id', + PRIMARY KEY (`receiver`,`topic`,`msg_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='消息消费状态表'; + +select @eventchecker_appconnId:=id from `dss_appconn` where `appconn_name` = 'eventchecker'; +delete from `dss_appconn_instance` where `appconn_id` = @eventchecker_appconnId; + +-- TODO 这里只适用于第一次安装时。如果是更新的话dss_appconn表不能先删除再插入,因为其他表如dss_workspace_appconn_role关联了appconn_id(不能变),需要使用update、alter语句更新 +INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) +VALUES ('eventchecker', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.dss.appconn.eventchecker.EventCheckerAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/eventchecker', ''); + +select @eventchecker_appconnId:=id from `dss_appconn` where `appconn_name` = 'eventchecker'; + +INSERT INTO `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`) +VALUES (@eventchecker_appconnId, 'DEV', 'eventchecker', '{\"msg.eventchecker.jdo.option.name\": \"msg\",\"msg.eventchecker.jdo.option.url\": \"EVENTCHECKER_JDBC_URL\",\"msg.eventchecker.jdo.option.username\": \"EVENTCHECKER_JDBC_USERNAME\",\"msg.eventchecker.jdo.option.password\": \"EVENTCHECKER_JDBC_PASSWORD\"}', ''); + +delete from dss_workflow_node where name in ('eventsender', 'eventreceiver'); +insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) +values('eventsender','eventchecker','linkis.appconn.eventchecker.eventsender','0','0','1','1','0','icons/eventsender.icon'); +insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) +values('eventreceiver','eventchecker','linkis.appconn.eventchecker.eventreceiver','0','0','1','1','0','icons/eventreceiver.icon'); + +select @eventsender_nodeId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.eventchecker.eventsender'; +select @eventreceiver_nodeId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.eventchecker.eventreceiver'; + +delete from `dss_workflow_node_to_group` where `node_id`=@eventchecker_nodeId; +delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@eventchecker_nodeId; + +-- 查找节点所属组的id +select @eventchecker_node_groupId:=id from `dss_workflow_node_group` where `name` = '信号节点'; + +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@eventsender_nodeId, @eventchecker_node_groupId); +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@eventreceiver_nodeId, @eventchecker_node_groupId); + +-- 考虑表中有的是重复记录,最好加上limit 1 +select @eventchecker_node_ui_lable_name_1:=id from `dss_workflow_node_ui` where `lable_name` = '节点名' limit 1; +select @eventchecker_node_ui_lable_name_2:=id from `dss_workflow_node_ui` where `lable_name` = '节点描述' limit 1; +select @eventchecker_node_ui_lable_name_3:=id from `dss_workflow_node_ui` where `lable_name` = '业务标签' limit 1; +select @eventchecker_node_ui_lable_name_4:=id from `dss_workflow_node_ui` where `lable_name` = '应用标签' limit 1; +select @eventchecker_node_ui_lable_name_5:=id from `dss_workflow_node_ui` where `lable_name` = '是否复用引擎' limit 1; +select @eventchecker_node_ui_lable_name_6:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.sender' limit 1; +select @eventchecker_node_ui_lable_name_7:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.topic' limit 1; +select @eventchecker_node_ui_lable_name_8:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.name' limit 1; +select @eventchecker_node_ui_lable_name_9:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.body' limit 1; +select @eventchecker_node_ui_lable_name_10:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.type' limit 1; + +select @receiver_node_ui_lable_name_1:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.type' order by id desc limit 1; +select @receiver_node_ui_lable_name_2:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.topic' order by id desc limit 1; +select @receiver_node_ui_lable_name_3:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.name' order by id desc limit 1; +-- eventreceiver ui +select @eventchecker_node_ui_lable_name_11:=id from `dss_workflow_node_ui` where `lable_name` = 'max.receive.hours' limit 1; +select @eventchecker_node_ui_lable_name_12:=id from `dss_workflow_node_ui` where `lable_name` = 'query.frequency' limit 1; +select @eventchecker_node_ui_lable_name_13:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.receiver' limit 1; +select @eventchecker_node_ui_lable_name_14:=id from `dss_workflow_node_ui` where `lable_name` = 'msg.savekey' limit 1; +select @eventchecker_node_ui_lable_name_15:=id from `dss_workflow_node_ui` where `lable_name` = 'only.receive.today' limit 1; + +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_4); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_5); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_6); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_7); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_8); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_9); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventsender_nodeId, @eventchecker_node_ui_lable_name_10); +-- eventreceiver +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_4); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_5); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @receiver_node_ui_lable_name_2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @receiver_node_ui_lable_name_3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @receiver_node_ui_lable_name_1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_11); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_12); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_13); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_14); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@eventreceiver_nodeId, @eventchecker_node_ui_lable_name_15); \ No newline at end of file diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/log4j.properties b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/log4j.properties new file mode 100644 index 000000000..ee8619595 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/log4j2.xml b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8c40a73e8 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/resources/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/eventchecker/execution/EventCheckerExecutionAction.scala b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/eventchecker/execution/EventCheckerExecutionAction.scala new file mode 100644 index 000000000..21c5ec71a --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/eventchecker/execution/EventCheckerExecutionAction.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.execution + +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.EventChecker +import com.webank.wedatasphere.dss.standard.app.development.listener.common.AbstractRefExecutionAction +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef + +class EventCheckerExecutionAction extends AbstractRefExecutionAction { + + private var response: ExecutionResponseRef = _ + + def setExecutionResponseRef(response: ExecutionResponseRef): Unit = + this.response = response + + def getExecutionResponseRef: ExecutionResponseRef = response + + private[this] var _saveKeyAndValue: String = _ + + def saveKeyAndValue: String = _saveKeyAndValue + + def saveKeyAndValue(value: String): Unit = { + _saveKeyAndValue = value + } + + private[this] var _eventType: String = "SEND" + + def eventType: String = _eventType + + def eventType(value: String): Unit = { + _eventType = value + } + + private[this] var _ec: EventChecker = _ + + def ec: EventChecker = _ec + + def setEc(value: EventChecker): Unit = { + _ec = value + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/eventchecker/execution/EventCheckerRefExecutionOperation.scala b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/eventchecker/execution/EventCheckerRefExecutionOperation.scala new file mode 100644 index 000000000..f6f75e6f3 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/eventchecker/execution/EventCheckerRefExecutionOperation.scala @@ -0,0 +1,154 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.execution + +import java.util.{Properties, UUID} + +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.EventChecker +import com.webank.wedatasphere.dss.standard.app.development.listener.common._ +import com.webank.wedatasphere.dss.standard.app.development.listener.core.{Killable, LongTermRefExecutionOperation, Procedure} +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef.ExecutionResponseRefBuilder +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.{AsyncExecutionResponseRef, ExecutionResponseRef, RefExecutionRequestRef} +import org.apache.commons.io.IOUtils +import org.apache.linkis.common.log.LogUtils +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.storage.LineRecord + + +class EventCheckerRefExecutionOperation + extends LongTermRefExecutionOperation[RefExecutionRequestRef.RefExecutionContextRequestRef] with Killable with Procedure { + + + override def progress(action: RefExecutionAction): Float = { + //temp set + 0.5f + } + + override def log(action: RefExecutionAction): String = { + action match { + case action: EventCheckerExecutionAction => + if (!action.getState.isCompleted) { + LogUtils.generateInfo("EventChecker is sending or waiting for message") + } else { + LogUtils.generateInfo("EventChecker successfully received or send message") + } + case _ => LogUtils.generateERROR("Error NodeExecutionAction for log") + } + } + + override def kill(action: RefExecutionAction): Boolean = { + action match { + case longTermAction: EventCheckerExecutionAction => + longTermAction.setKilledFlag(true) + longTermAction.setState(RefExecutionState.Killed) + true + case _ => { + logger.error("EventChecker kill failed for error action") + false + } + } + + } + + protected def putErrorMsg(errorMsg: String, t: Throwable, action: EventCheckerExecutionAction): EventCheckerExecutionAction = { + action.setExecutionResponseRef(new ExecutionResponseRefBuilder().setErrorMsg(errorMsg).setException(t).error()) + action + } + + override def submit(requestRef: RefExecutionRequestRef.RefExecutionContextRequestRef): RefExecutionAction = { + val nodeAction = new EventCheckerExecutionAction() + nodeAction.setId(UUID.randomUUID().toString) + import scala.collection.JavaConversions.mapAsScalaMap + val InstanceConfig = this.service.getAppInstance.getConfig + val scalaParams: scala.collection.mutable.Map[String, Object] = requestRef.getExecutionRequestRefContext.getRuntimeMap + val properties = new Properties() + InstanceConfig.foreach { record => + if(null == record._2) { + properties.put(record._1, "")} + else { + properties.put(record._1, record._2.toString) + } + } + scalaParams.foreach { case (key, value) => + if (key != null && value != null) properties.put(key, value.toString) + } + Utils.tryCatch{ + val ec = new EventChecker(properties, nodeAction) + ec.run() + nodeAction.setEc(ec) + } (t => { + logger.error("EventChecker run failed for " + t.getMessage, t) + putErrorMsg("EventChecker run failed!" + t.getMessage, t, nodeAction) + }) + nodeAction + } + + override def state(action: RefExecutionAction): RefExecutionState = { + action match { + case action: EventCheckerExecutionAction =>{ + action.getExecutionRequestRefContext.appendLog("EventCheck is running!") + if (action.getState.isCompleted) return action.getState + if (action.eventType.equals("RECEIVE")) { + Utils.tryCatch(action.ec.receiveMsg())(t => { + action.setState(RefExecutionState.Failed) + logger.error("EventChecker run failed for " + t.getMessage, t) + putErrorMsg("EventChecker run failed!" + t.getMessage, t, action) + false + }) + } + action.getState + } + case _ => { + logger.error("EventChecker run failed for error action") + RefExecutionState.Failed + } + } + } + + override def result(action: RefExecutionAction): ExecutionResponseRef = { + action match { + case action: EventCheckerExecutionAction => + if (action.getState.equals(RefExecutionState.Success)) { + val resultSetWriter = action.getExecutionRequestRefContext.createTextResultSetWriter() + var resultStr = "EventChecker runs successfully!" + if (action.saveKeyAndValue != null) { + resultStr = action.saveKeyAndValue + logger.info("EventChecker save receive value: " + resultStr) + } + Utils.tryFinally { + resultSetWriter.addMetaData(null) + resultSetWriter.addRecord(new LineRecord(resultStr)) + }(IOUtils.closeQuietly(resultSetWriter)) + new ExecutionResponseRefBuilder().success() + } else if(action.getExecutionResponseRef != null) action.getExecutionResponseRef + else new ExecutionResponseRefBuilder().error() + case _ => + new ExecutionResponseRefBuilder().error() + } + + } + + override def createAsyncResponseRef(requestRef: RefExecutionRequestRef.RefExecutionContextRequestRef, action: RefExecutionAction): AsyncExecutionResponseRef = { + action match { + case action: EventCheckerExecutionAction => + val response = super.createAsyncResponseRef(requestRef, action) + new AsyncExecutionResponseRef.Builder().setMaxLoopTime(action.ec.maxWaitTime) + .setAskStatePeriod(action.ec.queryFrequency).setAsyncExecutionResponseRef(response).build() + } + } + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/pom.xml b/dss-appconn/appconns/dss-schedulis-appconn/pom.xml new file mode 100644 index 000000000..a7af986cb --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/pom.xml @@ -0,0 +1,136 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-schedulis-appconn + + + + + + com.webank.wedatasphere.dss + dss-scheduler-appconn + ${dss.version} + + + + + com.webank.wedatasphere.dss + dss-origin-sso-integration-standard + ${dss.version} + + + linkis-common + org.apache.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + + + com.google.code.gson + gson + ${gson.version} + provided + + + com.webank.wedatasphere.dss + dss-contextservice + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/assembly/distribution.xml b/dss-appconn/appconns/dss-schedulis-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..36cd42577 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,76 @@ + + + + dss-schedulis-appconn + + dir + + true + schedulis + + + + + + lib + true + true + false + true + true + + + + + + ${basedir}/src/main/resources + + appconn.properties + + 0777 + / + unix + + + + ${basedir}/src/main/resources + + log4j.properties + log4j2.xml + + 0777 + conf + unix + + + + ${basedir}/src/main/resources + + init.sql + + 0777 + db + + + + + + + diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/SchedulisAppConn.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/SchedulisAppConn.java new file mode 100644 index 000000000..1fba3e480 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/SchedulisAppConn.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis; + +import com.webank.wedatasphere.dss.appconn.scheduler.AbstractSchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerStructureIntegrationStandard; +import com.webank.wedatasphere.dss.appconn.schedulis.standard.SchedulisStructureStandard; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ConversionIntegrationStandard; + +public class SchedulisAppConn extends AbstractSchedulerAppConn { + + public static final String SCHEDULIS_APPCONN_NAME = "Schedulis"; + + public ConversionIntegrationStandard getOrCreateConversionStandard() { + return super.getOrCreateConversionStandard(); + } + + @Override + public SchedulerStructureIntegrationStandard getOrCreateStructureStandard() { + return SchedulisStructureStandard.getInstance(); + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conf/AzkabanConf.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conf/AzkabanConf.java new file mode 100644 index 000000000..2277106ad --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conf/AzkabanConf.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.conf; + + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; + +public class AzkabanConf { + + public static final CommonVars DEFAULT_STORE_PATH = CommonVars.apply("wds.dss.appconn.scheduler.project.store.dir", "/appcom/tmp/wds/dss"); + public static final CommonVars LINKIS_VERSION = CommonVars.apply("wds.dss.appconn.scheduler.linkis.version", "1.0.0"); + public static final CommonVars REALESE_USER_FRESH_TIME = CommonVars.apply("wds.dss.appconn.scheduler.releaseUsers.fresh.interval", new TimeType("2m")); + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java new file mode 100644 index 000000000..92b73191d --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.constant; + +public class AzkabanConstant { + public final static String LINKIS_FLOW_VARIABLE_KEY = "flow.variable."; + public final static String AZKABAN_JOB_SUFFIX = ".job"; + public final static String AZKABAN_PROPERTIES_SUFFIX = ".properties"; + public final static String LINKIS_JOB_RESOURCES_KEY = "resources="; + public final static String ZAKABAN_DEPENDENCIES_KEY = "dependencies"; + public final static String JOB_TYPE = "type"; + public final static String JOB_LABELS = "labels"; + public final static String LINKIS_TYPE = "linkistype"; + public final static String JOB_COMMAND = "command"; + public final static String FLOW_CONTEXT_ID = "wds.linkis.flow.contextID="; + public final static String LINKIS_VERSION = "linkis.version"; + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/AzkabanWorkflowToRelSynchronizer.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/AzkabanWorkflowToRelSynchronizer.java new file mode 100644 index 000000000..1ae53f660 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/AzkabanWorkflowToRelSynchronizer.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.conversion; + +import com.webank.wedatasphere.dss.appconn.schedulis.SchedulisAppConn; +import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanConvertedRel; +import com.webank.wedatasphere.dss.appconn.schedulis.utils.SchedulisHttpUtils; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.DSSToRelConversionOperation; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSUploadAction; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.operation.WorkflowToRelSynchronizer; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.linkis.httpclient.request.BinaryBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +public class AzkabanWorkflowToRelSynchronizer implements WorkflowToRelSynchronizer { + + public static final Logger LOGGER = LoggerFactory.getLogger(AzkabanWorkflowToRelSynchronizer.class); + + private String projectUrl; + private DSSToRelConversionOperation dssToRelConversionOperation; + + public void init() { + String baseUrl = dssToRelConversionOperation.getConversionService().getAppInstance().getBaseUrl(); + this.projectUrl = baseUrl.endsWith("/") ? baseUrl + "manager": baseUrl + "/manager"; + } + + @Override + public void setDSSToRelConversionOperation(DSSToRelConversionOperation dssToRelConversionOperation) { + this.dssToRelConversionOperation = dssToRelConversionOperation; + init(); + } + + @Override + public void syncToRel(ConvertedRel convertedRel) { + String tmpSavePath; + AzkabanConvertedRel azkabanConvertedRel = (AzkabanConvertedRel) convertedRel; + try { + String projectPath = azkabanConvertedRel.getStorePath(); + tmpSavePath = ZipHelper.zip(projectPath); + //upload zip to Azkaban + uploadProject(azkabanConvertedRel.getDSSToRelConversionRequestRef().getWorkspace(), tmpSavePath, + azkabanConvertedRel.getDSSToRelConversionRequestRef().getDSSProject().getName(), azkabanConvertedRel.getDSSToRelConversionRequestRef().getUserName()); + } catch (Exception e) { + throw new DSSRuntimeException(90012, ExceptionUtils.getRootCauseMessage(e), e); + } + } + + private void uploadProject(Workspace workspace, String tmpSavePath, String projectName, String releaseUser) throws Exception { + + File file = new File(tmpSavePath); + InputStream inputStream = new FileInputStream(file); + try { + BinaryBody binaryBody = BinaryBody.apply("file",inputStream,file.getName(),"application/zip"); + List binaryBodyList =new ArrayList<>(); + binaryBodyList.add(binaryBody); + DSSUploadAction uploadAction = new DSSUploadAction(binaryBodyList); + uploadAction.getFormParams().put("ajax", "upload"); + uploadAction.getFormParams().put("project", projectName); + uploadAction.getParameters().put("project", projectName); + uploadAction.getParameters().put("ajax", "upload"); + uploadAction.setUrl(projectUrl); + SchedulisHttpUtils.getHttpResult(projectUrl, uploadAction, + dssToRelConversionOperation.getConversionService().getSSORequestService() + .createSSORequestOperation(SchedulisAppConn.SCHEDULIS_APPCONN_NAME), workspace); + } finally { + IOUtils.closeQuietly(inputStream); + } + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/NodeConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/NodeConverter.java new file mode 100644 index 000000000..605c61154 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/NodeConverter.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.conversion; + +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; + +public interface NodeConverter { + + String conversion(WorkflowNode workflowNode); + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/ProjectInfoWorkflowToRelConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/ProjectInfoWorkflowToRelConverter.java new file mode 100644 index 000000000..87df14a57 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/ProjectInfoWorkflowToRelConverter.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.conversion; + +import com.webank.wedatasphere.dss.appconn.schedulis.conf.AzkabanConf; +import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanConvertedRel; +import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanWorkflow; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ProjectPreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.operation.WorkflowToRelConverter; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + + +public class ProjectInfoWorkflowToRelConverter implements WorkflowToRelConverter { + + private static final Logger LOGGER = LoggerFactory.getLogger(ProjectInfoWorkflowToRelConverter.class); + + @Override + public ConvertedRel convertToRel(PreConversionRel rel) { + ProjectPreConversionRel projectPreConversionRel = (ProjectPreConversionRel) rel; + List nodeNames = getAllNodeName(projectPreConversionRel.getWorkflows()); + String repeatNodes = nodeNames.stream().collect(Collectors.groupingBy(Function.identity(), Collectors.counting())) + .entrySet().stream().filter(entry -> entry.getValue() > 1).map(Map.Entry::getKey).collect(Collectors.joining(", ")); + if (StringUtils.isNotEmpty(repeatNodes)) { + throw new DSSRuntimeException(80001, "重复的节点名称:" + repeatNodes); + } + AzkabanConvertedRel azkabanConvertedRel = new AzkabanConvertedRel(projectPreConversionRel); + //1. Assign a value to the storepath of azkabanschedulerproject. + assignStorePath(azkabanConvertedRel); + //2. The storepath of his rootflow is also assigned a value. + List workflows = projectPreConversionRel.getWorkflows(); + workflows.forEach(flow -> setRootFlowStorePath(azkabanConvertedRel.getStorePath(), flow)); + //3. Delete zip packages and folders that may not have been processed. + removeProjectStoreDirAndZip(azkabanConvertedRel); + return azkabanConvertedRel; + } + + private void setRootFlowStorePath(String projectStorePath, Workflow workflow){ + AzkabanWorkflow azkabanWorkflow = (AzkabanWorkflow) workflow; + azkabanWorkflow.setStorePath(projectStorePath + File.separator + workflow.getName()); + } + + private void assignStorePath(AzkabanConvertedRel rel) { + SimpleDateFormat dateFormat = new SimpleDateFormat(AzkabanConvertedRel.DATE_FORMAT); + Date date = new Date(); + String dateStr = dateFormat.format(date); + String userName = rel.getDSSToRelConversionRequestRef().getDSSProject().getUsername(); + String name = rel.getDSSToRelConversionRequestRef().getDSSProject().getName(); + String storePath = AzkabanConf.DEFAULT_STORE_PATH.getValue() + File.separator + userName + + File.separator + dateStr + File.separator +name; + rel.setStorePath(storePath); + } + + private void removeProjectStoreDirAndZip(AzkabanConvertedRel rel) { + String storePath = rel.getStorePath(); + File projectDir = new File(storePath); + try { + if (projectDir.exists()) { + LOGGER.info("exist project dir{} before publish ,now remove it", storePath); + FileUtils.deleteDirectory(projectDir); + } + String projectZip = projectDir.getParent() + File.separator + + rel.getDSSToRelConversionRequestRef().getDSSProject().getName() + ".zip"; + File zipFile = new File(projectZip); + if (zipFile.exists()) { + LOGGER.info("exist project zip{} before publish ,now remove it", projectZip); + zipFile.delete(); + } + } catch (Exception e) { + LOGGER.error("delete project dir or zip failed,reaseon:", e); + throw new DSSRuntimeException(90020, e.getMessage()); + } + } + + /** + * Get the names of all nodes directly from all flows without recursion. + */ + private List getAllNodeName(List workflows) { + List nodeNames = new ArrayList<>(); + workflows.forEach(flow -> flow.getWorkflowNodes().forEach(node -> nodeNames.add(node.getName()))); + return nodeNames; + } + + @Override + public int getOrder() { + return 5; + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanNodeRelConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanNodeRelConverter.java new file mode 100644 index 000000000..8e24ac580 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanNodeRelConverter.java @@ -0,0 +1,113 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.conversion; + +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.appconn.schedulis.constant.AzkabanConstant; +import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanWorkflow; +import com.webank.wedatasphere.dss.appconn.schedulis.linkisjob.LinkisJobConverter; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ProjectPreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.operation.WorkflowToRelConverter; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.util.List; + + +public class WorkflowToAzkbanNodeRelConverter implements WorkflowToRelConverter { + + public static final Logger LOGGER = LoggerFactory.getLogger(WorkflowToAzkbanNodeRelConverter.class); + + private NodeConverter nodeConverter; + + public WorkflowToAzkbanNodeRelConverter() { + nodeConverter = ClassUtils.getInstanceOrDefault(NodeConverter.class, new LinkisJobConverter()); + } + + @Override + public ConvertedRel convertToRel(PreConversionRel rel) { + ((ProjectPreConversionRel) rel).getWorkflows().forEach(this::convertNode); + return (ConvertedRel) rel; + } + + private void convertNode(Workflow workflow) { + workflow.getWorkflowNodes().forEach(DSSExceptionUtils.handling(workflowNode -> { + String nodeStorePath = getNodeStorePath(((AzkabanWorkflow)workflow).getStorePath(), workflowNode); + writeNodeToJobLocal(workflowNode, nodeStorePath); + writeNodeResourcesToLocal(workflowNode, nodeStorePath); + })); + if(workflow.getChildren() != null) { + workflow.getChildren().forEach(flow -> convertNode((Workflow) flow)); + } + } + + private String getNodeStorePath(String flowStorePath, WorkflowNode schedulerNode) { + return flowStorePath + File.separator + "jobs" + File.separator + schedulerNode.getName(); + } + + private void writeNodeToJobLocal(WorkflowNode workflowNode, String storePath) throws DSSErrorException { + FileOutputStream os = null; + try { + File jobDirFile = new File(storePath); + FileUtils.forceMkdir(jobDirFile); + File jobFile = new File(storePath,workflowNode.getName() + AzkabanConstant.AZKABAN_JOB_SUFFIX); + jobFile.createNewFile(); + String nodeString = nodeConverter.conversion(workflowNode); + os = FileUtils.openOutputStream(jobFile,true); + os.write(nodeString.getBytes()); + }catch (Exception e){ + LOGGER.error("write AppConnNode to jobLocal failed,reason:",e); + throw new DSSErrorException(90017,e.getMessage()); + } finally { + IOUtils.closeQuietly(os); + } + } + + private void writeNodeResourcesToLocal(WorkflowNode workflowNode, String storePath) throws DSSErrorException { + List nodeResources = workflowNode.getDSSNode().getResources(); + if(nodeResources == null || nodeResources.isEmpty()) {return;} + FileOutputStream os = null; + try { + File jobFile = new File(storePath,workflowNode.getName() + AzkabanConstant.AZKABAN_JOB_SUFFIX); + String nodeResourceString = AzkabanConstant.LINKIS_JOB_RESOURCES_KEY + new Gson().toJson(nodeResources); + os = FileUtils.openOutputStream(jobFile,true); + os.write(nodeResourceString.getBytes()); + }catch (Exception e){ + LOGGER.error("write nodeResources to local failed,reason:",e); + throw new DSSErrorException(90018,e.getMessage()); + }finally { + IOUtils.closeQuietly(os); + } + } + + @Override + public int getOrder() { + return 100; + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanRelConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanRelConverter.java new file mode 100644 index 000000000..a413d9d93 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanRelConverter.java @@ -0,0 +1,142 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.conversion; + +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.appconn.schedulis.constant.AzkabanConstant; +import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanConvertedRel; +import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanWorkflow; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.workflow.common.entity.Flow; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.operation.WorkflowToRelConverter; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import java.io.File; +import java.io.FileOutputStream; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class WorkflowToAzkbanRelConverter implements WorkflowToRelConverter { + + public static final Logger LOGGER = LoggerFactory.getLogger(WorkflowToAzkbanRelConverter.class); + + @Override + public ConvertedRel convertToRel(PreConversionRel rel) { + AzkabanConvertedRel azkabanConvertedRel = (AzkabanConvertedRel) rel; + azkabanConvertedRel.getWorkflows().forEach(DSSExceptionUtils.handling(workflow -> { + //1. Set sub flow and node storage paths + String flowStorePath = ((AzkabanWorkflow) workflow).getStorePath(); + if (workflow.getChildren() != null) { + workflow.getChildren().forEach(flow -> setFlowStorePath(flowStorePath, flow)); + } + // 2. Processing resources, generating files, and so on. + writeWorkflowFiles(workflow, azkabanConvertedRel.getStorePath()); + })); + return azkabanConvertedRel; + } + + private void writeWorkflowFiles(Flow workflow, String projectStorePath) throws DSSErrorException { + AzkabanWorkflow flow = (AzkabanWorkflow) workflow; + writeFlowResourcesToLocal(flow, projectStorePath); + writeFlowPropertiesToLocal(flow); + if (workflow.getChildren() != null) { + workflow.getChildren().forEach(DSSExceptionUtils.handling(f -> writeWorkflowFiles(f, projectStorePath))); + } + } + + private void setFlowStorePath(String flowStorePath, Flow workflow) { + AzkabanWorkflow azkabanWorkflow = (AzkabanWorkflow) workflow; + azkabanWorkflow.setStorePath(flowStorePath + File.separator + "subFlows" + File.separator + azkabanWorkflow.getName()); + if (workflow.getChildren() != null) { + workflow.getChildren().forEach(flow -> setFlowStorePath(azkabanWorkflow.getStorePath(), flow)); + } + } + + private void writeFlowResourcesToLocal(AzkabanWorkflow flow, String projectStorePath) throws DSSErrorException { + List flowResources = flow.getFlowResources(); + FileOutputStream os = null; + try { + String storePath = flow.getStorePath(); + File flowDir = new File(storePath); + FileUtils.forceMkdir(flowDir); + if (flowResources == null || flowResources.isEmpty()) { + return; + } + String flowResourceStringPrefix = getFlowResourceStringPrefix(projectStorePath, storePath); + String flowResourceString = flowResourceStringPrefix + new Gson().toJson(flowResources) + "\n"; + File projectResourcesFile = new File(projectStorePath, "project.properties"); + os = FileUtils.openOutputStream(projectResourcesFile, true); + os.write(flowResourceString.getBytes()); + } catch (Exception e) { + LOGGER.error("write FlowResources to local failed,reason:", e); + throw new DSSErrorException(90006, e.getMessage()); + } finally { + IOUtils.closeQuietly(os); + } + } + + private String getFlowResourceStringPrefix(String projectStorePath, String storePath) { + String substring = storePath.substring(projectStorePath.length() + 1); + String prefix = substring.replaceAll("\\" + File.separator + "subFlows" + "\\" + File.separator, "."); + return "flow." + prefix + "_.resources="; + } + + private void writeFlowPropertiesToLocal(AzkabanWorkflow flow) throws DSSErrorException { + List> flowProperties = flow.getFlowProperties(); + if (flowProperties == null || flowProperties.isEmpty()) { + return; + } + FileOutputStream os = null; + try { + String storePath = flow.getStorePath(); + File flowPrpsFile = new File(storePath, flow.getName() + AzkabanConstant.AZKABAN_PROPERTIES_SUFFIX); + flowPrpsFile.createNewFile(); + os = FileUtils.openOutputStream(flowPrpsFile, true); + StringBuilder stringBuilder = new StringBuilder(); + flowProperties.forEach(p -> p.forEach((k, v) -> { + stringBuilder.append(AzkabanConstant.LINKIS_FLOW_VARIABLE_KEY + k + "=" + v + "\n"); + })); + // update by peaceWong add contextID to Flow properties + String contextID = flow.getContextID(); + if (StringUtils.isNotBlank(contextID)) { + contextID = contextID.replace("\\", "/"); + LOGGER.info("after replace contextID is {}", contextID); + stringBuilder.append(AzkabanConstant.FLOW_CONTEXT_ID + contextID + "\n"); + } + // update end + os.write(stringBuilder.toString().getBytes()); + } catch (Exception e) { + LOGGER.error("write flowProperties to local faailed,reason:", e); + throw new DSSErrorException(90007, e.getMessage()); + } finally { + IOUtils.closeQuietly(os); + } + } + + @Override + public int getOrder() { + return 10; + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanConvertedRel.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanConvertedRel.java new file mode 100644 index 000000000..6030e0634 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanConvertedRel.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.entity; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.ProjectToRelConversionRequestRef; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ProjectPreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ProjectPreConversionRelImpl; + +public class AzkabanConvertedRel extends ProjectPreConversionRelImpl implements ConvertedRel { + + private String storePath; + public static final String DATE_FORMAT = "yyyyMMddHHmmss"; + + public AzkabanConvertedRel(ProjectPreConversionRel rel) { + setWorkflows(rel.getWorkflows()); + setDSSToRelConversionRequestRef(rel.getDSSToRelConversionRequestRef()); + } + + public AzkabanConvertedRel(AzkabanConvertedRel rel) { + this((ProjectPreConversionRel) rel); + storePath = rel.getStorePath(); + } + + @Override + public ProjectToRelConversionRequestRef getDSSToRelConversionRequestRef() { + return (ProjectToRelConversionRequestRef) super.getDSSToRelConversionRequestRef(); + } + + public String getStorePath() { + return storePath; + } + + public void setStorePath(String storePath) { + this.storePath = storePath; + } + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanUserEntity.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanUserEntity.java new file mode 100644 index 000000000..d4e839fd6 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanUserEntity.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.entity; + +public class AzkabanUserEntity { + private String id; + private String text; + private String username; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanWorkflow.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanWorkflow.java new file mode 100644 index 000000000..1387db767 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/entity/AzkabanWorkflow.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.entity; + +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowWithContextImpl; + +public class AzkabanWorkflow extends WorkflowWithContextImpl { + + private String storePath; + + public String getStorePath() { + return storePath; + } + + public void setStorePath(String storePath) { + this.storePath = storePath; + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/AzkabanSubFlowJobTuning.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/AzkabanSubFlowJobTuning.java new file mode 100644 index 000000000..b083be21a --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/AzkabanSubFlowJobTuning.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.linkisjob; + +public class AzkabanSubFlowJobTuning implements LinkisJobTuning { + + @Override + public LinkisJob tuningJob(LinkisJob job) { + job.setType("flow"); + job.setLinkistype(null); + job.getConf().put("flow.name",job.getName() + "_"); + return job; + } + + @Override + public boolean ifJobCantuning(String nodeType) { + return "workflow.subflow".equals(nodeType); + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java new file mode 100644 index 000000000..5c68f9676 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java @@ -0,0 +1,86 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.linkisjob; + +import java.util.Map; + +public class LinkisJob { + private String name; + private String type; + private String linkistype; + private String proxyUser; + private String dependencies; + private Map conf; + private String command; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getLinkistype() { + return linkistype; + } + + public void setLinkistype(String linkistype) { + this.linkistype = linkistype; + } + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + + public String getDependencies() { + return dependencies; + } + + public void setDependencies(String dependencies) { + this.dependencies = dependencies; + } + + public Map getConf() { + return conf; + } + + public void setConf(Map conf) { + this.conf = conf; + } + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java new file mode 100644 index 000000000..3cede173c --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java @@ -0,0 +1,130 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.linkisjob; + +import com.webank.wedatasphere.dss.appconn.scheduler.utils.SchedulerConf; +import com.webank.wedatasphere.dss.appconn.schedulis.conf.AzkabanConf; +import com.webank.wedatasphere.dss.appconn.schedulis.constant.AzkabanConstant; +import com.webank.wedatasphere.dss.appconn.schedulis.conversion.NodeConverter; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.workflow.core.constant.WorkflowConstant; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import org.apache.commons.lang.StringUtils; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class LinkisJobConverter implements NodeConverter { + + private LinkisJobTuning[] linkisJobTunings; + + public LinkisJobConverter(){ + this.linkisJobTunings = new LinkisJobTuning[]{new AzkabanSubFlowJobTuning()}; + } + + @Override + public String conversion(WorkflowNode workflowNode){ + return baseConversion(workflowNode); + } + + private String baseConversion(WorkflowNode workflowNode){ + LinkisJob job = new LinkisJob(); + job.setConf(new HashMap<>()); + job.setName(workflowNode.getName()); + convertHead(workflowNode,job); + convertDependencies(workflowNode,job); + convertProxyUser(workflowNode,job); + convertConfiguration(workflowNode,job); + convertJobCommand(workflowNode,job); + Arrays.stream(linkisJobTunings).forEach(t ->{ + if(t.ifJobCantuning(workflowNode.getNodeType())) { + t.tuningJob(job); + } + }); + return convertJobToString(job); + } + + private String convertJobToString(LinkisJob job){ + HashMap map = new HashMap<>(8); + map.put(AzkabanConstant.LINKIS_VERSION, AzkabanConf.LINKIS_VERSION.getValue()); + map.put(AzkabanConstant.JOB_TYPE,job.getType()); + map.put(AzkabanConstant.LINKIS_TYPE,job.getLinkistype()); + map.put(AzkabanConstant.ZAKABAN_DEPENDENCIES_KEY,job.getDependencies()); + map.put(WorkflowConstant.PROXY_USER,job.getProxyUser()); + map.put(AzkabanConstant.JOB_COMMAND,job.getCommand()); + Map labels = new HashMap<>(1); + labels.put("route", SchedulerConf.JOB_LABEL.getValue()); + map.put(AzkabanConstant.JOB_LABELS, DSSCommonUtils.COMMON_GSON.toJson(labels)); + map.putAll(job.getConf()); + StringBuilder stringBuilder = new StringBuilder(); + map.forEach((k,v)->{ + if(v != null) { + //for value contains "\n" + v = v.replace("\n", ";"); + stringBuilder.append(k).append("=").append(v).append("\n"); + } + }); + return stringBuilder.toString(); + } + + private void convertHead(WorkflowNode workflowNode, LinkisJob job){ + job.setType("linkis"); + job.setLinkistype(workflowNode.getNodeType()); + } + + private void convertDependencies(WorkflowNode workflowNode, LinkisJob job){ + List dependencys = workflowNode.getDSSNode().getDependencys(); + if(dependencys != null && !dependencys.isEmpty()) { + StringBuilder dependencies = new StringBuilder(); + dependencys.forEach(d -> dependencies.append(d).append(",")); + job.setDependencies(dependencies.substring(0,dependencies.length()-1)); + } + } + + private void convertProxyUser(WorkflowNode workflowNode, LinkisJob job){ + String userProxy = workflowNode.getDSSNode().getUserProxy(); + if(!StringUtils.isEmpty(userProxy)) { + job.setProxyUser(userProxy); + } + } + + private void convertConfiguration(WorkflowNode workflowNode, LinkisJob job){ + Map params = workflowNode.getDSSNode().getParams(); + if (params != null && !params.isEmpty()) { + Map> configuration = (Map>) params.get("configuration"); + String confprefix = "node.conf."; + configuration.forEach((k,v)-> { + if(null!=v) { + v.forEach((k2, v2) -> { + if(null !=v2) {job.getConf().put(confprefix + k + "." + k2, v2.toString());} + }); + } + }); + } + + } + + private void convertJobCommand(WorkflowNode workflowNode, LinkisJob job){ + Map jobContent = workflowNode.getDSSNode().getJobContent(); + if(jobContent != null) { + jobContent.remove("jobParams"); + job.setCommand(DSSCommonUtils.COMMON_GSON.toJson(jobContent)); + } + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobTuning.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobTuning.java new file mode 100644 index 000000000..5f76cd64b --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobTuning.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.linkisjob; + +public interface LinkisJobTuning { + + LinkisJob tuningJob(LinkisJob job); + + boolean ifJobCantuning(String nodeType); +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectCreationOperation.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectCreationOperation.java new file mode 100644 index 000000000..c01ca93d6 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectCreationOperation.java @@ -0,0 +1,124 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.operation; + +import com.webank.wedatasphere.dss.appconn.schedulis.SchedulisAppConn; +import com.webank.wedatasphere.dss.appconn.schedulis.service.AzkabanUserService; +import com.webank.wedatasphere.dss.appconn.schedulis.service.SchedulisProjectService; +import com.webank.wedatasphere.dss.appconn.schedulis.utils.AzkabanUtils; +import com.webank.wedatasphere.dss.appconn.schedulis.utils.SchedulisHttpUtils; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectCreationOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.HashMap; +import java.util.Map; + +public class SchedulisProjectCreationOperation + extends AbstractStructureOperation + implements ProjectCreationOperation { + + private String projectUrl; + + private String managerUrl; + + @Override + protected String getAppConnName() { + return SchedulisAppConn.SCHEDULIS_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + this.projectUrl = getBaseUrl().endsWith("/") ? getBaseUrl() + "manager" : getBaseUrl() + "/manager"; + managerUrl = getBaseUrl().endsWith("/") ? getBaseUrl() + "manager" : getBaseUrl() + "/manager"; + } + + @Override + public ProjectResponseRef createProject(DSSProjectContentRequestRef.DSSProjectContentRequestRefImpl requestRef) throws ExternalOperationFailedException { + logger.info("begin to create project in schedulis, project name is {}.", requestRef.getDSSProject().getName()); + if (CollectionUtils.isNotEmpty(requestRef.getDSSProjectPrivilege().getReleaseUsers())) { + // 先校验运维用户是否存在于 Schedulis,如果不存在,则不能成功创建工程。 + requestRef.getDSSProjectPrivilege().getReleaseUsers().forEach(releaseUser -> { + if (!AzkabanUserService.containsUser(releaseUser, getBaseUrl(), ssoRequestOperation, requestRef.getWorkspace())) { + throw new ExternalOperationFailedException(100323, "当前设置的发布用户: " + releaseUser + ", 在 Schedulis 系统中不存在,请联系 Schedulis 管理员创建该用户!"); + } + }); + } + Map params = new HashMap<>(3); + params.put("action", "create"); + params.put("name", requestRef.getDSSProject().getName()); + params.put("description", requestRef.getDSSProject().getDescription()); + try { + String entStr = SchedulisHttpUtils.getHttpPostResult(projectUrl, params, ssoRequestOperation, requestRef.getWorkspace()); + logger.error("新建工程 {}, Schedulis 返回的信息是 {}.", requestRef.getName(), entStr); + String message = AzkabanUtils.handleAzkabanEntity(entStr); + if (!"success".equals(message)) { + throw new ExternalOperationFailedException(90008, "Schedulis 新建工程失败, 原因: " + message); + } + } catch (final Exception t) { + logger.error("Failed to create project!", t); + return ProjectResponseRef.newExternalBuilder().error(t); + } + // 绑定权限 + if(CollectionUtils.isNotEmpty(requestRef.getDSSProjectPrivilege().getReleaseUsers())) { + ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl updateRequestRef = new ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl() + .setWorkspace(requestRef.getWorkspace()).setUserName(requestRef.getUserName()).setDSSProject(requestRef.getDSSProject()) + .setDSSProjectPrivilege(requestRef.getDSSProjectPrivilege()).setAddedDSSProjectPrivilege(requestRef.getDSSProjectPrivilege()); + ((SchedulisProjectService) service).getProjectUpdateOperation().updateProject(updateRequestRef); + } + Long projectId = null; + try { + projectId = getSchedulisProjectId(requestRef.getDSSProject().getName(), requestRef); + } catch (Exception e) { + DSSExceptionUtils.dealWarnException(60051, "failed to get project id.", e, + ExternalOperationFailedException.class); + } + + return ProjectResponseRef.newExternalBuilder().setRefProjectId(projectId).success(); + } + + /** + * Get project ID. + */ + public Long getSchedulisProjectId(String projectName, + DSSProjectContentRequestRef.DSSProjectContentRequestRefImpl requestRef) { + + Map params = new HashMap<>(2); + params.put("ajax", "getProjectId"); + params.put("project", projectName); + long projectId = 0L; + try { + String content = SchedulisHttpUtils.getHttpGetResult(this.managerUrl, params, ssoRequestOperation, requestRef.getWorkspace()); + logger.info("Get Schedulis project id return str is {}.", content); + Map map = DSSCommonUtils.COMMON_GSON.fromJson(content, Map.class); + projectId = DSSCommonUtils.parseToLong(map.get("projectId")); + } catch (final Throwable t) { + DSSExceptionUtils.dealWarnException(60051, "failed to fetch project id in schedulis.", t, + ExternalOperationFailedException.class); + } + return projectId; + } + + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectDeletionOperation.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectDeletionOperation.java new file mode 100644 index 000000000..548bc4464 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectDeletionOperation.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.operation; + +import com.webank.wedatasphere.dss.appconn.schedulis.SchedulisAppConn; +import com.webank.wedatasphere.dss.appconn.schedulis.utils.SchedulisHttpUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +import java.util.HashMap; +import java.util.Map; + +public class SchedulisProjectDeletionOperation + extends AbstractStructureOperation + implements ProjectDeletionOperation { + + private String managerUrl; + + @Override + protected String getAppConnName() { + return SchedulisAppConn.SCHEDULIS_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + managerUrl = getBaseUrl().endsWith("/") ? getBaseUrl() + "manager" : + getBaseUrl() + "/manager"; + } + + @Override + public ResponseRef deleteProject(RefProjectContentRequestRef.RefProjectContentRequestRefImpl projectRef) throws ExternalOperationFailedException { + try { + Map params = new HashMap<>(2); + params.put("project", projectRef.getProjectName()); + params.put("delete", "true"); + String responseContent = SchedulisHttpUtils.getHttpGetResult(this.managerUrl, params, ssoRequestOperation, projectRef.getWorkspace()); + logger.info("delete Schedulis Project with response: {}.", responseContent); + } catch (Exception e){ + DSSExceptionUtils.dealWarnException(60052, "failed to delete project in Schedulis.", e, + ExternalOperationFailedException.class); + } + return ProjectResponseRef.newExternalBuilder().success(); + } + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java new file mode 100644 index 000000000..89e363539 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java @@ -0,0 +1,57 @@ +package com.webank.wedatasphere.dss.appconn.schedulis.operation; + +import com.webank.wedatasphere.dss.appconn.schedulis.SchedulisAppConn; +import com.webank.wedatasphere.dss.appconn.schedulis.utils.SchedulisHttpUtils; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectSearchOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +import java.util.HashMap; +import java.util.Map; + +public class SchedulisProjectSearchOperation + extends AbstractStructureOperation + implements ProjectSearchOperation { + + private String queryUrl; + + @Override + public ProjectResponseRef searchProject(RefProjectContentRequestRef.RefProjectContentRequestRefImpl requestRef) throws ExternalOperationFailedException { + logger.info("begin to search Schedulis project , projectName is {}.", requestRef.getProjectName()); + + Map params = new HashMap<>(2); + params.put("project", requestRef.getProjectName()); + params.put("ajax", "fetchprojectflows"); + try { + String responseBody = SchedulisHttpUtils.getHttpGetResult(queryUrl, params, ssoRequestOperation, requestRef.getWorkspace()); + logger.info("responseBody from Schedulis is: {}.", responseBody); + Map map = DSSCommonUtils.COMMON_GSON.fromJson(responseBody, Map.class); + String errorInfo = (String) map.get("error"); + if (errorInfo != null && (errorInfo.contains("Project " + requestRef.getProjectName() + " doesn't exist") + //schedulis已删除但未永久删除的项目返回这个 + || errorInfo.contains("Permission denied. Need READ access"))) { + return ProjectResponseRef.newExternalBuilder().success(); + } else if (errorInfo != null) { + return ProjectResponseRef.newExternalBuilder().error(errorInfo); + } + return ProjectResponseRef.newExternalBuilder().setRefProjectId(DSSCommonUtils.parseToLong(map.get("projectId"))).success(); + } catch (Exception e) { + throw new ExternalOperationFailedException(90117, "Failed to search Schedulis project name!", e); + } + } + + @Override + protected String getAppConnName() { + return SchedulisAppConn.SCHEDULIS_APPCONN_NAME; + } + + @Override + public void init() { + super.init(); + queryUrl = mergeBaseUrl("manager"); + } + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectUpdateOperation.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectUpdateOperation.java new file mode 100644 index 000000000..e2195cdf8 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectUpdateOperation.java @@ -0,0 +1,116 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.operation; + +import com.webank.wedatasphere.dss.appconn.schedulis.SchedulisAppConn; +import com.webank.wedatasphere.dss.appconn.schedulis.service.AzkabanUserService; +import com.webank.wedatasphere.dss.appconn.schedulis.utils.SchedulisHttpUtils; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.HashMap; +import java.util.Map; + +public class SchedulisProjectUpdateOperation + extends AbstractStructureOperation + implements ProjectUpdateOperation { + + private String managerUrl; + + @Override + public void init() { + super.init(); + managerUrl = mergeBaseUrl("manager"); + } + + @Override + public ResponseRef updateProject(ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl projectRef) { + if (CollectionUtils.isNotEmpty(projectRef.getDSSProjectPrivilege().getReleaseUsers())) { + // 先校验运维用户是否存在于 Schedulis,如果不存在,则不能成功创建工程。 + projectRef.getDSSProjectPrivilege().getReleaseUsers().forEach(releaseUser -> { + if (!AzkabanUserService.containsUser(releaseUser, getBaseUrl(), ssoRequestOperation, projectRef.getWorkspace())) { + throw new ExternalOperationFailedException(100323, "当前设置的发布用户: " + releaseUser + ", 在 Schedulis 系统中不存在,请联系 Schedulis 管理员创建该用户!"); + } + }); + } + // 更新description + syncProjectDesc(projectRef); + // 往 Schedulis 增加新增的用户 + addProjectPrivilege(projectRef); + // 往 Schedulis 删除已去掉的用户 + removeRelProjectPrivilege(projectRef); + return ResponseRef.newInternalBuilder().success(); + } + + private void addProjectPrivilege(ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl projectRef) { + String projectName = projectRef.getProjectName(); + if (projectRef.getAddedDSSProjectPrivilege() == null || + CollectionUtils.isEmpty(projectRef.getAddedDSSProjectPrivilege().getReleaseUsers())) { + return; + } + projectRef.getAddedDSSProjectPrivilege().getReleaseUsers().forEach(releaseUser -> { + String userId = AzkabanUserService.getUserId(releaseUser, getBaseUrl(), ssoRequestOperation, projectRef.getWorkspace()); + Map params = new HashMap<>(8); + params.put("project", projectName); + params.put("userId", userId); + params.put("ajax", "ajaxAddProjectUserPermission"); + params.put("permissions[admin]", "false"); + params.put("permissions[read]", "true"); + params.put("permissions[write]", "true"); + params.put("permissions[execute]", "true"); + params.put("permissions[schedule]", "true"); + String response = SchedulisHttpUtils.getHttpGetResult(managerUrl, params, ssoRequestOperation, projectRef.getWorkspace()); + logger.info("for project {} add a accessUser {} response is {}.", + projectName, releaseUser, response); + }); + } + + private void removeRelProjectPrivilege(ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl projectRef) { + String projectName = projectRef.getProjectName(); + if (projectRef.getRemovedDSSProjectPrivilege() == null || + CollectionUtils.isEmpty(projectRef.getRemovedDSSProjectPrivilege().getReleaseUsers())) { + return; + } + projectRef.getRemovedDSSProjectPrivilege().getReleaseUsers().forEach(accessUser -> { + Map params = new HashMap<>(3); + params.put("project", projectName); + params.put("userId", accessUser); + params.put("ajax", "ajaxRemoveProjectAdmin"); + String response = SchedulisHttpUtils.getHttpGetResult(managerUrl, params, ssoRequestOperation, projectRef.getWorkspace()); + logger.info("for project {} remove a accessUser {} response is {}.", projectName, accessUser, response); + }); + } + + private void syncProjectDesc(ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl projectRef) { + String projectName = projectRef.getProjectName(); + Map params = new HashMap<>(); + params.put("project", projectName); + params.put("ajax", "changeDescription"); + params.put("description", projectRef.getDSSProject().getDescription()); + String response = SchedulisHttpUtils.getHttpGetResult(managerUrl, params, ssoRequestOperation, projectRef.getWorkspace()); + logger.info("for project {} update description, response is {}.", projectName, response); + } + + @Override + protected String getAppConnName() { + return SchedulisAppConn.SCHEDULIS_APPCONN_NAME; + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/parser/AzkabanWorkflowParser.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/parser/AzkabanWorkflowParser.java new file mode 100644 index 000000000..0c0377c1f --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/parser/AzkabanWorkflowParser.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.parser; + +import com.google.gson.JsonObject; +import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanWorkflow; +import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNodeEdge; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNodeImpl; +import com.webank.wedatasphere.dss.workflow.core.json2flow.parser.WorkflowParser; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.beanutils.BeanUtils; + +public class AzkabanWorkflowParser implements WorkflowParser { + + @Override + public Workflow parse(JsonObject flowJson, Workflow workflow) { + AzkabanWorkflow azkabanWorkflow = new AzkabanWorkflow(); + try { + BeanUtils.copyProperties(azkabanWorkflow, workflow); + } catch (Exception e) { + throw new DSSRuntimeException(91500, "Copy workflow fields failed!", e); + } + return addEndNodeForFlowName(azkabanWorkflow); + } + + private AzkabanWorkflow addEndNodeForFlowName(AzkabanWorkflow flow) { + DSSNodeDefault endNode = new DSSNodeDefault(); + List endNodeList = getFlowEndJobList(flow); + if(flow.getRootFlow()){ + endNode.setId(flow.getName()); + endNode.setName(flow.getName()); + }else{ + endNode.setId(flow.getName() + "_"); + endNode.setName(flow.getName() + "_"); + } + endNode.setNodeType("linkis.control.empty"); + Map jobContentMap = new HashMap<>(); + endNode.setJobContent(jobContentMap); + if (!endNodeList.isEmpty()) { + if(endNodeList.size() == 1 ) { + if(endNodeList.get(0).getName().equals(flow.getName())){ + return flow; + } + } + endNodeList.forEach(tmpNode -> endNode.addDependency(tmpNode.getName())); + WorkflowNode azkabanSchedulerNode = new WorkflowNodeImpl(); + azkabanSchedulerNode.setDSSNode(endNode); + flow.getWorkflowNodes().add(azkabanSchedulerNode); + } + return flow; + } + + private List getFlowEndJobList(AzkabanWorkflow flow) { + List res = new ArrayList<>(); + for (WorkflowNode job : flow.getWorkflowNodes()) { + int flag = 0; + for (WorkflowNodeEdge link : flow.getWorkflowNodeEdges()) { + if (job.getId().equals(link.getDSSEdge().getSource())) { + flag = 1; + } + } + if (flag == 0) { + res.add(job); + } + } + return res; + } + + @Override + public int getOrder() { + return 100; + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/AzkabanUserService.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/AzkabanUserService.java new file mode 100644 index 000000000..da9f85a1a --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/AzkabanUserService.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.service; + +import com.webank.wedatasphere.dss.appconn.schedulis.conf.AzkabanConf; +import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanUserEntity; +import com.webank.wedatasphere.dss.appconn.schedulis.utils.SchedulisHttpUtils; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class AzkabanUserService { + + private static final Map> schedulisUserMap = new HashMap<>(2); + private static final Map userMapLastModified = new ConcurrentHashMap<>(2); + + private static final Logger LOGGER = LoggerFactory.getLogger(AzkabanUserService.class); + + private static void requestUserId(String baseUrl, SSORequestOperation ssoRequestOperation, Workspace workspace) { + if (userMapLastModified.containsKey(baseUrl) && + System.currentTimeMillis() - userMapLastModified.get(baseUrl) < AzkabanConf.REALESE_USER_FRESH_TIME.getValue().toLong()) { + return; + } + LOGGER.info("try to update all releaseUsers from Schedulis url {}.", baseUrl); + Map params = new HashMap<>(3); + params.put("page", "1"); + params.put("pageSize", "10000"); + params.put("ajax", "loadSystemUserSelectData"); + String finalUrl = !baseUrl.endsWith("/") ? (baseUrl + "/" + "system") : baseUrl + "system"; + try { + String response = SchedulisHttpUtils.getHttpGetResult(finalUrl, params, ssoRequestOperation, workspace); + Map map = DSSCommonUtils.COMMON_GSON.fromJson(response, Map.class); + if (map.get("systemUserList") instanceof List) { + if (!schedulisUserMap.containsKey(baseUrl)) { + synchronized (schedulisUserMap) { + if (!schedulisUserMap.containsKey(baseUrl)) { + schedulisUserMap.put(baseUrl, new ArrayList<>()); + } + } + } + List entityList = schedulisUserMap.get(baseUrl); + List newEntityList = ((List) map.get("systemUserList")).stream().map(e -> + DSSCommonUtils.COMMON_GSON.fromJson(e.toString(), AzkabanUserEntity.class) + ).collect(Collectors.toList()); + synchronized (entityList) { + entityList.clear(); + entityList.addAll(newEntityList); + } + } + } catch (Exception e) { + LOGGER.error("update all releaseUsers from Schedulis url {} failed.", baseUrl, e); + } + userMapLastModified.put(baseUrl, System.currentTimeMillis()); + } + + public static boolean containsUser(String releaseUser, String baseUrl, + SSORequestOperation ssoRequestOperation, Workspace workspace) { + Supplier supplier = () -> schedulisUserMap.containsKey(baseUrl) && + schedulisUserMap.get(baseUrl).stream().anyMatch(entity -> entity.getUsername().equals(releaseUser)); + if (!supplier.get()) { + requestUserId(baseUrl, ssoRequestOperation, workspace); + } + return supplier.get(); + } + + public static String getUserId(String user, String baseUrl, + SSORequestOperation ssoRequestOperation, Workspace workspace) { + if(containsUser(user, baseUrl, ssoRequestOperation, workspace)) { + return schedulisUserMap.get(baseUrl).stream().filter(entity -> entity.getUsername().equals(user)).findAny().get().getId(); + } else { + throw new ExternalOperationFailedException(10823, "Not exists user in Schedulis " + user); + } + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/SchedulisProjectService.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/SchedulisProjectService.java new file mode 100644 index 000000000..91a27252e --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/SchedulisProjectService.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.service; + +import com.webank.wedatasphere.dss.appconn.schedulis.operation.SchedulisProjectCreationOperation; +import com.webank.wedatasphere.dss.appconn.schedulis.operation.SchedulisProjectDeletionOperation; +import com.webank.wedatasphere.dss.appconn.schedulis.operation.SchedulisProjectSearchOperation; +import com.webank.wedatasphere.dss.appconn.schedulis.operation.SchedulisProjectUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.*; + +public class SchedulisProjectService extends ProjectService { + + public SchedulisProjectService(){ + } + + @Override + protected ProjectCreationOperation createProjectCreationOperation() { + return new SchedulisProjectCreationOperation(); + } + + @Override + protected ProjectUpdateOperation createProjectUpdateOperation() { + return new SchedulisProjectUpdateOperation(); + } + + @Override + protected ProjectDeletionOperation createProjectDeletionOperation() { + return new SchedulisProjectDeletionOperation(); + } + + @Override + protected ProjectSearchOperation createProjectSearchOperation() { + return new SchedulisProjectSearchOperation(); + } + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/standard/SchedulisStructureStandard.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/standard/SchedulisStructureStandard.java new file mode 100644 index 000000000..c9c4006c2 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/standard/SchedulisStructureStandard.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.standard; + +import com.webank.wedatasphere.dss.appconn.scheduler.AbstractSchedulerStructureIntegrationStandard; +import com.webank.wedatasphere.dss.appconn.schedulis.service.SchedulisProjectService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; + + +/** + * Schedulis's engineering integration specification is a singleton. + */ +public class SchedulisStructureStandard extends AbstractSchedulerStructureIntegrationStandard { + + private volatile static SchedulisStructureStandard instance; + + private SchedulisStructureStandard(){ + } + + public static SchedulisStructureStandard getInstance(){ + if(instance == null){ + synchronized (SchedulisStructureStandard.class){ + if (instance == null){ + instance = new SchedulisStructureStandard(); + } + } + } + return instance; + } + + @Override + protected ProjectService createProjectService() { + return new SchedulisProjectService(); + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/AzkabanUtils.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/AzkabanUtils.java new file mode 100644 index 000000000..6d0588d48 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/AzkabanUtils.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.utils; + +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import org.apache.commons.lang.StringUtils; + +import java.util.Map; + +public class AzkabanUtils { + + public static String handleAzkabanEntity(String entityString) { + if(StringUtils.isNotBlank(entityString)){ + if(entityString.startsWith("{") && entityString.endsWith("}")){ + Map resMap = DSSCommonUtils.COMMON_GSON.fromJson(entityString, Map.class); + if(resMap.containsKey("error")){ + return (String)resMap.get("error"); + } + } + if(entityString.contains("
") && entityString.contains("
")){ + return "The SSO login status is invalid.Please refresh the browser and log in again!"; + } + } + + Object object = DSSCommonUtils.COMMON_GSON.fromJson(entityString, Object.class); + String status = null; + String message = null; + if (object instanceof Map) { + Map map = (Map) object; + if (map.get("status") != null) { + status = map.get("status").toString(); + } + if (StringUtils.isNotEmpty(status)) { + if (null != map.get("message")) { + message = map.get("message").toString(); + } + } + if ("error".equalsIgnoreCase(status)) { + return message; + } + } + return "success"; + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/SchedulisHttpUtils.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/SchedulisHttpUtils.java new file mode 100644 index 000000000..12cc31e7e --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/SchedulisHttpUtils.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.utils; + +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSGetAction; +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSHttpAction; +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSPostAction; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.linkis.httpclient.request.HttpAction; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static com.webank.wedatasphere.dss.appconn.schedulis.SchedulisAppConn.SCHEDULIS_APPCONN_NAME; + + +public class SchedulisHttpUtils { + + private final static Logger logger = LoggerFactory.getLogger(SchedulisHttpUtils.class); + + public static SSOUrlBuilderOperation getSSORequestOperation(String url, Workspace workspace) { + SSOUrlBuilderOperation ssoUrlBuilderOperation = SSOHelper.createSSOUrlBuilderOperation(workspace); + ssoUrlBuilderOperation.setAppName(SCHEDULIS_APPCONN_NAME); + ssoUrlBuilderOperation.setReqUrl(url); + return ssoUrlBuilderOperation; + } + + public static String getHttpResult(String url, + DSSHttpAction action, + SSORequestOperation ssoRequestOperation, + Workspace workspace) { + SSOUrlBuilderOperation ssoUrlBuilderOperation = getSSORequestOperation(url, workspace); + action.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); + HttpResult previewResult = ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, action); + if (previewResult.getStatusCode() == 200 || previewResult.getStatusCode() == 0) { + return previewResult.getResponseBody(); + } else { + logger.error("request Schedulis failed, responseBody is {}.", previewResult.getResponseBody()); + throw new ExternalOperationFailedException(50063, "request Schedulis failed."); + } + } + + public static String getHttpGetResult(String url, + Map params, + SSORequestOperation ssoRequestOperation, + Workspace workspace) { + DSSGetAction getAction = new DSSGetAction(); + getAction.getParameters().putAll(params); + return getHttpResult(url, getAction, ssoRequestOperation, workspace); + } + + public static String getHttpPostResult(String url, + Map params, + SSORequestOperation ssoRequestOperation, + Workspace workspace) { + DSSPostAction getAction = new DSSPostAction(); + getAction.getFormParams().putAll(params); + return getHttpResult(url, getAction, ssoRequestOperation, workspace); + } + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/appconn.properties b/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/appconn.properties new file mode 100644 index 000000000..e1f7094b2 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/appconn.properties @@ -0,0 +1,20 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +wds.dss.appconn.scheduler.project.store.dir=/appcom/tmp/wds/scheduler + + + diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/init.sql b/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/init.sql new file mode 100644 index 000000000..a52fde468 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/init.sql @@ -0,0 +1,19 @@ +-- 适用于第一次安装时 +select @schedulis_appconnId:=id from `dss_appconn` where `appconn_name` = 'schedulis'; +delete from `dss_appconn_instance` where `appconn_id` = @schedulis_appconnId; + +delete from dss_appconn where appconn_name = "schedulis"; +INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) +VALUES ('schedulis', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.dss.appconn.schedulis.SchedulisAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/schedulis', ''); + +select @schedulis_appconnId:=id from `dss_appconn` where `appconn_name` = 'schedulis'; + +insert into `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`) +values(@schedulis_appconnId,'DEV','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/','{"reqUri":""}','/manager'); + +-- 看appconn组件是要归属于哪个菜单 +select @schedulis_menuId:=id from dss_workspace_menu where name = "生产运维"; + +delete from dss_workspace_menu_appconn where title_en = "Schedulis"; +INSERT INTO `dss_workspace_menu_appconn` (`appconn_id`, `menu_id`, `title_en`, `title_cn`, `desc_en`, `desc_cn`, `labels_en`, `labels_cn`, `is_active`, `access_button_en`, `access_button_cn`, `manual_button_en`, `manual_button_cn`, `manual_button_url`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`, `image`) +VALUES(@schedulis_appconnId, @schedulis_menuId,'Schedulis','Schedulis','empty desc','empty desc','scheduling, workflow','调度,工作流','1','enter Schedulis','进入Schedulis','user manual','用户手册','manual_url','diaoduxitong-logo',NULL,NULL,NULL,NULL,NULL,'diaoduxitong-icon'); diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/log4j.properties b/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/log4j.properties new file mode 100644 index 000000000..ee8619595 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/log4j2.xml b/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8c40a73e8 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-appconn/appconns/dss-scriptis-appconn/pom.xml b/dss-appconn/appconns/dss-scriptis-appconn/pom.xml new file mode 100644 index 000000000..fef31c8f2 --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/pom.xml @@ -0,0 +1,69 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-scriptis-appconn + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/assembly/distribution.xml b/dss-appconn/appconns/dss-scriptis-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..94b137259 --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,50 @@ + + + + dss-scriptis-appconn + + dir + + true + scriptis + + + + ${basedir}/src/main/resources + + init.sql + + 0777 + db + + + + ${basedir}/src/main/icons + + * + + 0777 + icons + + + + + + diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/connector.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/connector.icon new file mode 100644 index 000000000..ea7c69e77 --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/connector.icon @@ -0,0 +1 @@ +connectorCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/hql.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/hql.icon new file mode 100644 index 000000000..fc85faf66 --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/hql.icon @@ -0,0 +1 @@ + hqlCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/jdbc.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/jdbc.icon new file mode 100644 index 000000000..31270ec8b --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/jdbc.icon @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/pyspark.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/pyspark.icon new file mode 100644 index 000000000..abdabfc01 --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/pyspark.icon @@ -0,0 +1 @@ +pysparkCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/python.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/python.icon new file mode 100644 index 000000000..ed1ed43f6 --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/python.icon @@ -0,0 +1 @@ +pythonCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/scala.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/scala.icon new file mode 100644 index 000000000..3b25b493c --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/scala.icon @@ -0,0 +1 @@ +ScalaCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/shell.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/shell.icon new file mode 100644 index 000000000..954da701b --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/shell.icon @@ -0,0 +1 @@ +shellCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/sql.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/sql.icon new file mode 100644 index 000000000..db6069052 --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/sql.icon @@ -0,0 +1 @@ +sqlCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/subFlow.icon b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/subFlow.icon new file mode 100644 index 000000000..feae0c4e2 --- /dev/null +++ b/dss-appconn/appconns/dss-scriptis-appconn/src/main/icons/subFlow.icon @@ -0,0 +1 @@ +subflowCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-scriptis-appconn/src/main/resources/init.sql b/dss-appconn/appconns/dss-scriptis-appconn/src/main/resources/init.sql new file mode 100644 index 000000000..e69de29bb diff --git a/dss-appconn/appconns/dss-sendemail-appconn/pom.xml b/dss-appconn/appconns/dss-sendemail-appconn/pom.xml new file mode 100644 index 000000000..780c124bd --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/pom.xml @@ -0,0 +1,36 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../pom.xml + + 4.0.0 + + dss-sendemail-appconn + pom + + + sendemail-appconn-core + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml new file mode 100644 index 000000000..394da4a18 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml @@ -0,0 +1,153 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../pom.xml + + 4.0.0 + + dss-sendemail-appconn-core + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + compile + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + compile + + + + com.webank.wedatasphere.dss + dss-development-process-standard-execution + ${dss.version} + + + + org.apache.linkis + linkis-storage + ${linkis.version} + provided + true + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + true + + + + org.springframework + spring-context-support + 5.2.5.RELEASE + + + javax.mail + mail + 1.4 + + + + org.apache.linkis + linkis-cs-client + ${linkis.version} + + + org.apache.httpcomponents + httpclient + 4.5.13 + compile + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.properties + **/application.yml + **/bootstrap.yml + **/log4j2.xml + + + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/assembly/distribution.xml b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/assembly/distribution.xml new file mode 100644 index 000000000..2519296b9 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/assembly/distribution.xml @@ -0,0 +1,82 @@ + + + + dss-sendemail-appconn + + dir + + true + sendemail + + + + + + lib + true + true + false + true + true + + + + + + ${basedir}/src/main/resources + + appconn.properties + + 0777 + / + unix + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/src/main/icons + + * + + 0777 + icons + + + + ${basedir}/src/main/resources + + init.sql + + 0777 + db + + + + + + diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/icons/sendemail.icon b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/icons/sendemail.icon new file mode 100644 index 000000000..208519eeb --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/icons/sendemail.icon @@ -0,0 +1 @@ +sendemailCreated with Sketch. \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/EmailDevelopmentIntegrationStandard.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/EmailDevelopmentIntegrationStandard.java new file mode 100644 index 000000000..a615c60fb --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/EmailDevelopmentIntegrationStandard.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail; + +import com.webank.wedatasphere.dss.appconn.sendemail.service.EmailExecutionService; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.*; +import com.webank.wedatasphere.dss.standard.app.development.standard.OnlyExecutionDevelopmentStandard; + +/** + * @author allenlliu + * @date 2021/10/27 11:34 + */ +public class EmailDevelopmentIntegrationStandard extends OnlyExecutionDevelopmentStandard { + + @Override + public void close() { + } + + @Override + protected RefExecutionService createRefExecutionService() { + return new EmailExecutionService(); + } + + @Override + public void init() { + + } + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/SendEmailAppConn.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/SendEmailAppConn.java new file mode 100644 index 000000000..a44cbf7d2 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/SendEmailAppConn.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail; + +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.appconn.core.impl.AbstractAppConn; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExecutionService; +import com.webank.wedatasphere.dss.standard.app.development.service.RefExecutionService; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.development.standard.OnlyExecutionDevelopmentStandard; + +public class SendEmailAppConn extends AbstractAppConn implements OnlyDevelopmentAppConn { + + private EmailDevelopmentIntegrationStandard standard; + + @Override + protected void initialize() { + standard = new EmailDevelopmentIntegrationStandard(); + } + + @Override + public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() { + return standard; + } +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/conf/SendEmailAppConnInstanceConfiguration.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/conf/SendEmailAppConnInstanceConfiguration.java new file mode 100644 index 000000000..6194f30a3 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/conf/SendEmailAppConnInstanceConfiguration.java @@ -0,0 +1,116 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.conf; + +import com.webank.wedatasphere.dss.appconn.sendemail.email.EmailGenerator; +import com.webank.wedatasphere.dss.appconn.sendemail.email.EmailSender; +import com.webank.wedatasphere.dss.appconn.sendemail.email.generate.MultiContentEmailGenerator; +import com.webank.wedatasphere.dss.appconn.sendemail.email.sender.SpringJavaEmailSender; +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.EmailContentGenerator; +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.EmailContentParser; +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.generator.MultiEmailContentGenerator; +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser.FileEmailContentParser$; +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser.HtmlEmailContentParser$; +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser.PictureEmailContentParser$; +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser.TableEmailContentParser$; +import com.webank.wedatasphere.dss.appconn.sendemail.hook.SendEmailRefExecutionHook; +import com.webank.wedatasphere.dss.standard.common.utils.AppStandardClassUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; + +public class SendEmailAppConnInstanceConfiguration { + + private static final Logger logger = LoggerFactory.getLogger(SendEmailAppConnInstanceConfiguration.class); + + private static final EmailGenerator EMAIL_GENERATOR = new MultiContentEmailGenerator(); + + private static final EmailSender EMAIL_SENDER = createEmailSender(); + + private static final EmailContentGenerator[] EMAIL_CONTENT_GENERATOR = createEmailContentGenerators(); + + private static final EmailContentParser[] emailContentParsers = createEmailContentParsers(); + + private static final SendEmailRefExecutionHook[] sendEmailRefExecutionHooks = createSendEmailRefExecutionHooks(); + + private static EmailSender createEmailSender() { + String emailSenderClassName = SendEmailAppConnConfiguration.EMAIL_SENDER_CLASS().getValue(); + try { + logger.info("Use user config EmailSender by conf:{}", emailSenderClassName); + return (EmailSender)SendEmailAppConnInstanceConfiguration.class.getClassLoader().loadClass(emailSenderClassName).newInstance(); + } catch (Exception e) { + logger.warn("{} can not be instanced, use SpringJavaEmailSender by default.", emailSenderClassName, e); + return new SpringJavaEmailSender(); + } + } + + private static EmailContentGenerator[] createEmailContentGenerators() { + return new EmailContentGenerator[] {new MultiEmailContentGenerator()}; + } + + private static EmailContentParser[] createEmailContentParsers() { + return new EmailContentParser[] {FileEmailContentParser$.MODULE$, + HtmlEmailContentParser$.MODULE$, PictureEmailContentParser$.MODULE$, TableEmailContentParser$.MODULE$}; + } + + private static SendEmailRefExecutionHook[] createSendEmailRefExecutionHooks() { + String hookClasses = SendEmailAppConnConfiguration.EMAIL_HOOK_CLASSES().getValue(); + logger.info("Use email hook class: {}", hookClasses); + return Arrays.stream(hookClasses.split(",")).map(clazz -> { + SendEmailRefExecutionHook sendEmailRefExecutionHook = null; + try { + sendEmailRefExecutionHook = (SendEmailRefExecutionHook)SendEmailAppConnInstanceConfiguration.class.getClassLoader().loadClass(clazz).newInstance(); + logger.info("Get hook class instance is : {}", sendEmailRefExecutionHook.getClass().getName()); + } catch (InstantiationException e) { + logger.warn("{} can not be instanced", clazz, e); + } catch (IllegalAccessException e) { + logger.warn("{} can not be instanced", clazz, e); + } catch (ClassNotFoundException e) { + logger.warn("{} can not be instanced", clazz, e); + } + return sendEmailRefExecutionHook; + }).filter(hook -> null!= hook).toArray(SendEmailRefExecutionHook[]::new); + } + + public static EmailSender getEmailSender() { + return EMAIL_SENDER; + } + + public static void init(){ + logger.info("init SendEmailAppConnInstanceConfiguration"); + } + + public static EmailGenerator getEmailGenerator() { + return EMAIL_GENERATOR; + } + + + public static EmailContentGenerator[] getEmailContentGenerators() { + return EMAIL_CONTENT_GENERATOR; + } + + public static EmailContentParser[] getEmailContentParsers() { + return emailContentParsers; + } + + public static SendEmailRefExecutionHook[] getSendEmailRefExecutionHooks() { + return sendEmailRefExecutionHooks; + } + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/Email.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/Email.java new file mode 100644 index 000000000..cfd53b8ff --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/Email.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email; + +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.Attachment; + +public interface Email { + + String getContent(); + void setContent(String content); + + Attachment[] getAttachments(); + void setAttachments(Attachment[] attachments); + + String getSubject(); + void setSubject(String subject); + + String getFrom(); + void setFrom(String from); + + String getTo(); + void setTo(String to); + + String getCc(); + void setCc(String cc); + + String getBcc(); + void setBcc(String bcc); + + String getEmailType(); + void setEmailType(String emailType); +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/sender/AbstractEmailSender.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/sender/AbstractEmailSender.java new file mode 100644 index 000000000..71158b690 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/sender/AbstractEmailSender.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email.sender; + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email; +import com.webank.wedatasphere.dss.appconn.sendemail.email.EmailSender; +import org.apache.linkis.common.utils.Utils; +import scala.runtime.BoxedUnit; + +import java.util.Map; +import java.util.concurrent.Future; + +public abstract class AbstractEmailSender implements EmailSender { + + @Override + public void init(Map properties) { + } + + @Override + public Future sendAsync(Email email) { + return Utils.defaultScheduler().submit(() -> { + send(email); + return BoxedUnit.UNIT; + }); + } + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/sender/SpringJavaEmailSender.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/sender/SpringJavaEmailSender.java new file mode 100644 index 000000000..5ee863af9 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/sender/SpringJavaEmailSender.java @@ -0,0 +1,112 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email.sender; + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email; +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.Attachment; +import com.webank.wedatasphere.dss.appconn.sendemail.exception.EmailSendFailedException; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.mail.javamail.JavaMailSenderImpl; +import org.springframework.mail.javamail.MimeMessageHelper; + +import java.util.Base64; +import javax.mail.internet.MimeMessage; +import javax.mail.util.ByteArrayDataSource; +import java.util.Map; +import java.util.Properties; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import static com.webank.wedatasphere.dss.appconn.sendemail.conf.SendEmailAppConnConfiguration.*; + +public class SpringJavaEmailSender extends AbstractEmailSender { + + private static final Logger logger = LoggerFactory.getLogger(SpringJavaEmailSender.class); + + private JavaMailSenderImpl javaMailSender = new JavaMailSenderImpl(); + + @Override + public void init(Map properties) { + Properties prop = new Properties(); + prop.put("mail.smtp.auth", EMAIL_SMTP_AUTH().getValue(properties)); + prop.put("mail.smtp.starttls.enable", EMAIL_SMTP_STARTTLS_ENABLE().getValue(properties)); + prop.put("mail.smtp.starttls.required", EMAIL_SMTP_STARTTLS_REQUIRED().getValue(properties)); + prop.put("mail.smtp.ssl.enable", EMAIL_SMTP_SSL_ENABLED().getValue(properties)); + prop.put("mail.smtp.timeout", EMAIL_SMTP_TIMEOUT().getValue(properties)); + javaMailSender.setJavaMailProperties(prop); + BiConsumer, CommonVars> setProp = (consumer, c) -> { + String value = c.getValue(properties); + if(StringUtils.isBlank(value)) { + throw new ExternalOperationFailedException(84002, "The value of " + c.key() + " in sendEmail AppConn is null, please set it in appconn.properties of sendEmail AppConn."); + } else { + consumer.accept(value); + } + }; + setProp.accept(javaMailSender::setHost, EMAIL_HOST()); + javaMailSender.setPort(EMAIL_PORT().getValue(properties)); + setProp.accept(javaMailSender::setUsername, EMAIL_USERNAME()); + setProp.accept(javaMailSender::setPassword, EMAIL_PASSWORD()); + javaMailSender.setProtocol(EMAIL_PROTOCOL().getValue(properties)); + } + + @Override + public void send(Email email) throws EmailSendFailedException { + logger.info("Begin to send Email({}).", email.getSubject()); + try { + javaMailSender.send(parseToMimeMessage(email)); + } catch (Exception e) { + logger.error("Send email failed: ", e); + EmailSendFailedException ex = new EmailSendFailedException(80001, "Send email failed!"); + ex.initCause(e); + throw ex; + } + logger.info("Send Email({}) succeed.", email.getSubject()); + } + + private MimeMessage parseToMimeMessage(Email email) { + MimeMessage message = javaMailSender.createMimeMessage(); + try { + MimeMessageHelper messageHelper = new MimeMessageHelper(message, true); + if (StringUtils.isBlank(javaMailSender.getUsername())) { + messageHelper.setFrom(DEFAULT_EMAIL_FROM().getValue()); + } else { + messageHelper.setFrom(javaMailSender.getUsername()); + } + messageHelper.setSubject(email.getSubject()); + messageHelper.setTo(email.getTo()); + if (StringUtils.isNotBlank(email.getCc())) { + messageHelper.setCc(email.getCc()); + } + if (StringUtils.isNotBlank(email.getBcc())) { + messageHelper.setBcc(email.getBcc()); + } + messageHelper.setText(email.getContent(), true); + for (Attachment attachment : email.getAttachments()) { + messageHelper.addInline(attachment.getName(), new ByteArrayDataSource(Base64.getMimeDecoder().decode(attachment.getBase64Str()), attachment.getMediaType())); + messageHelper.addAttachment(attachment.getName(), new ByteArrayDataSource(Base64.getMimeDecoder().decode(attachment.getBase64Str()), attachment.getMediaType())); + } + + } catch (Exception e) { + logger.error("Send mail failed", e); + } + return message; + } +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContent.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContent.java new file mode 100644 index 000000000..c69b0d5f0 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContent.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent; + +public interface EmailContent { + + T getContent(); + + void setContent(T content); + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/exception/EmailSendFailedException.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/exception/EmailSendFailedException.java new file mode 100644 index 000000000..58aebf238 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/exception/EmailSendFailedException.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.exception; + + +import org.apache.linkis.common.exception.ErrorException; + +public class EmailSendFailedException extends ErrorException { + public EmailSendFailedException(int errCode, String desc) { + super(errCode, desc); + } +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/EmailInfo.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/EmailInfo.java new file mode 100644 index 000000000..7060ac95d --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/EmailInfo.java @@ -0,0 +1,139 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.hook; + +import java.util.Map; + +public class EmailInfo { + private String formId; + private String user; + private String dssProject; + private String widgets; + private String cc; + private String to; + private String bcc; + private String status; + private String priority; + private String alertList; + private int alertInterval = 60; + private String requestCreatedate; + private Map widgetColumns; + + public String getFormId() { + return formId; + } + + public void setFormId(String formId) { + this.formId = formId; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getDssProject() { + return dssProject; + } + + public void setDssProject(String dssProject) { + this.dssProject = dssProject; + } + + public String getWidgets() { + return widgets; + } + + public void setWidgets(String widgets) { + this.widgets = widgets; + } + + public String getCc() { + return cc; + } + + public void setCc(String cc) { + this.cc = cc; + } + + public String getTo() { + return to; + } + + public void setTo(String to) { + this.to = to; + } + + public String getBcc() { + return bcc; + } + + public void setBcc(String bcc) { + this.bcc = bcc; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getPriority() { + return priority; + } + + public void setPriority(String priority) { + this.priority = priority; + } + + public String getAlertList() { + return alertList; + } + + public void setAlertList(String alertList) { + this.alertList = alertList; + } + + public int getAlertInterval() { + return alertInterval; + } + + public void setAlertInterval(int alertInterval) { + this.alertInterval = alertInterval; + } + + public String getRequestCreatedate() { + return requestCreatedate; + } + + public void setRequestCreatedate(String requestCreatedate) { + this.requestCreatedate = requestCreatedate; + } + + public Map getWidgetColumns() { + return widgetColumns; + } + + public void setWidgetColumns(Map widgetColumns) { + this.widgetColumns = widgetColumns; + } +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/HttpClientUtil.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/HttpClientUtil.java new file mode 100644 index 000000000..f2bf89f9d --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/HttpClientUtil.java @@ -0,0 +1,446 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.hook; + + +import org.apache.http.Consts; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.SocketTimeoutException; +import java.net.URLEncoder; +import java.security.cert.CertificateException; +import java.text.SimpleDateFormat; +import java.util.*; + +@SuppressWarnings("all") +public final class HttpClientUtil { + private final static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); + public final static int connectTimeout = 5000; + private static PoolingHttpClientConnectionManager connManager = null; + private static CloseableHttpClient httpclient = null; + + private static TrustManager trustAllManager = new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + }; + + static { + httpclient = HttpClients.createDefault(); + } + + + public static String postForm(String url, int timeout, Map headerMap, List paramsList, String encoding){ + HttpPost post = new HttpPost(url); + try { + if(headerMap != null){ + for(Map.Entry entry : headerMap.entrySet()){ + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + //post.setHeader("Content-type", "application/json"); + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(timeout) + .setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + + post.setEntity(new UrlEncodedFormEntity(paramsList, encoding)); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if(entity != null){ + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if(entity != null){ + entity.getContent().close(); + } + } + } finally { + if(response != null){ + response.close(); + } + } + } catch (Exception e) { + throw new RuntimeException("invoke http post error!",e); + } finally { + post.releaseConnection(); + } + return ""; + } + + public static String postJsonBody(String url, int timeout, Map headerMap, + String paraData, String encoding) { + + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Map.Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + StringEntity jsonEntity = new StringEntity(paraData, ContentType.APPLICATION_JSON); + post.setConfig(requestConfig); + post.setEntity(jsonEntity); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("postJsonBody error: "+e.getMessage()); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("postJsonBody Exception: "+e.getMessage()); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + @SuppressWarnings("deprecation") + public static String invokeGet(String url, Map params, String encode, int connectTimeout, + int soTimeout) { + String responseString = null; + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + + StringBuilder sb = new StringBuilder(); + sb.append(url); + int i = 0; + if (params != null) { + for (Map.Entry entry : params.entrySet()) { + if (i == 0 && !url.contains("?")) { + sb.append("?"); + } else { + sb.append("&"); + } + sb.append(entry.getKey()); + sb.append("="); + String value = entry.getValue(); + try { + sb.append(URLEncoder.encode(value, "UTF-8")); + } catch (UnsupportedEncodingException e) { + logger.warn("encode http get params error, value is " + value, e); + sb.append(URLEncoder.encode(value)); + } + i++; + } + } + HttpGet get = new HttpGet(sb.toString()); + get.setConfig(requestConfig); + try { + CloseableHttpResponse response = httpclient.execute(get); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + responseString = EntityUtils.toString(entity, encode); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]get response error, url:%s", sb.toString()), e); + return responseString; + } finally { + if (response != null) { + response.close(); + } + } + // System.out.println(String.format("[HttpUtils Get]Debug url:%s , + // response string %s:", sb.toString(), responseString)); + } catch (SocketTimeoutException e) { + logger.error(String.format("[HttpUtils Get]invoke get timout error, url:%s", sb.toString()), e); + return responseString; + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]invoke get error, url:%s", sb.toString()), e); + } finally { + get.releaseConnection(); + } + return responseString; + } + + /** + * HTTPS请求,默认超时为5S + * + * @param reqURL + * @param params + * @return + */ + public static String connectPostHttps(String reqURL, Map params) { + + String responseContent = null; + HttpPost httpPost = new HttpPost(reqURL); + try { + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + List formParams = new ArrayList(); + httpPost.setEntity(new UrlEncodedFormEntity(formParams, Consts.UTF_8)); + httpPost.setConfig(requestConfig); + // 绑定到请求 Entry + for (Map.Entry entry : params.entrySet()) { + formParams.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); + } + CloseableHttpResponse response = httpclient.execute(httpPost); + try { + // 执行POST请求 + HttpEntity entity = response.getEntity(); // 获取响应实体 + try { + if (null != entity) { + responseContent = EntityUtils.toString(entity, Consts.UTF_8); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + logger.info("requestURI : " + httpPost.getURI() + ", responseContent: " + responseContent); + } catch (ClientProtocolException e) { + logger.error("ClientProtocolException", e); + } catch (IOException e) { + logger.error("IOException", e); + } finally { + httpPost.releaseConnection(); + } + return responseContent; + + } + + class Test { + String v; + String k; + + public String getV() { + return v; + } + + public void setV(String v) { + this.v = v; + } + + public String getK() { + return k; + } + + public void setK(String k) { + this.k = k; + } + + } + + // 随机4位数 + public static String getRandomValue() { + String str = "0123456789"; + StringBuilder sb = new StringBuilder(4); + for (int i = 0; i < 4; i++) { + char ch = str.charAt(new Random().nextInt(str.length())); + sb.append(ch); + } + return sb.toString(); + } + + // 当前时间到秒 + public static String getTimestamp() { + + Date date = new Date(); + String timestamp = String.valueOf(date.getTime() / 1000); + return timestamp; + } + + // 当前时间到秒 + public static String getNowDate() { + Date date = new Date(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); + return sdf.format(date); + } + + public static String postJsonBody2(String url, int timeout, Map headerMap, + List paramsList, String encoding) { + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Map.Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(paramsList, encoding); + post.setEntity(entity); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + public static String postJsonBody3(String url, int timeout, Map headerMap, + Map paramsList, String encoding) { + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Map.Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + //JSONArray jsonArray = JSONArray.fromObject(paramsList); + //post.setEntity(new StringEntity(jsonArray.get(0).toString(), encoding)); + post.setEntity(new StringEntity(null, encoding)); + //logger.info("successfully start post Json Body url{},params ", url,jsonArray.get(0).toString()); + logger.info("successfully start post Json Body url{},params ", url,null); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + public static String executeGet(String url) + { + String rtnStr = ""; + HttpGet httpGet = new HttpGet(url); + try { + HttpResponse httpResponse = httpclient.execute(httpGet); + //获得返回的结果 + rtnStr = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + e.printStackTrace(); + } finally { + httpGet.releaseConnection(); + } + return rtnStr; + } + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/HttpResponseModel.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/HttpResponseModel.java new file mode 100644 index 000000000..a6965b114 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/HttpResponseModel.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.hook; + +public abstract class HttpResponseModel { + private String method; + private int status; + private String message; + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/WidgetMetaData.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/WidgetMetaData.java new file mode 100644 index 000000000..1e268538f --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/hook/WidgetMetaData.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.hook; + +import java.util.List; +import java.util.Map; + +public class WidgetMetaData extends HttpResponseModel{ + public static class Meta{ + private String name; + private String updated; + private String columns; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUpdated() { + return updated; + } + + public void setUpdated(String updated) { + this.updated = updated; + } + + public String getColumns() { + return columns; + } + + public void setColumns(String columns) { + this.columns = columns; + } + } + + + public static class Data{ + private String projectName; + private List widgetsMetaData; + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public List getWidgetsMetaData() { + return widgetsMetaData; + } + + public void setWidgetsMetaData(List widgetsMetaData) { + this.widgetsMetaData = widgetsMetaData; + } + } + + private Data data; + + public Data getData() { + return data; + } + + public void setData(Data data) { + this.data = data; + } +} + diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/service/EmailExecutionService.java b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/service/EmailExecutionService.java new file mode 100644 index 000000000..6316f4597 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/service/EmailExecutionService.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.service; + +import com.webank.wedatasphere.dss.appconn.sendemail.SendEmailRefExecutionOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExecutionService; + +/** + * @author allenlliu + * @date 2021/10/27 11:39 + */ +public class EmailExecutionService extends AbstractRefExecutionService { + + @Override + public RefExecutionOperation createRefExecutionOperation() { + return new SendEmailRefExecutionOperation(); + } +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/resources/appconn.properties b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/resources/appconn.properties new file mode 100644 index 000000000..30ea40102 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/resources/appconn.properties @@ -0,0 +1,34 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# The following properties must be set in. +wds.dss.appconn.email.host= +wds.dss.appconn.email.port= +# wds.dss.appconn.email.protocol=smtp +wds.dss.appconn.email.username= +wds.dss.appconn.email.password= + +# The following properties have default value. +# wds.dss.appconn.email.smtp.auth=true +# wds.dss.appconn.email.smtp.starttls.enable=true +# wds.dss.appconn.email.smtp.starttls.required=true +# wds.dss.appconn.email.smtp.ssl.enable=true +# wds.dss.appconn.email.smtp.timeout=25000 + + + diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/resources/init.sql b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/resources/init.sql new file mode 100644 index 000000000..75654adc7 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/resources/init.sql @@ -0,0 +1,51 @@ +-- TODO 这里只适用于第一次安装时。如果是更新的话dss_appconn表不能先删除再插入,因为其他表如dss_workspace_appconn_role关联了appconn_id(不能变),需要使用update、alter语句更新 +select @sendemail_appconnId:=id from `dss_appconn` where `appconn_name` = 'sendemail'; +delete from `dss_appconn_instance` where `appconn_id` = @sendemail_appconnId; + +INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) +VALUES ('sendemail', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.dss.appconn.sendemail.SendEmailAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/sendemail', ''); + +select @sendemail_appconnId:=id from `dss_appconn` where `appconn_name` = 'sendemail'; + +INSERT INTO `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`) +VALUES (@sendemail_appconnId, 'DEV', 'sendemail', '{"email.host":"EMAIL_HOST","email.port":"EMAIL_PORT","email.username":"EMAIL_USERNAME","email.password":"EMAIL_PASSWORD","email.protocol":"EMAIL_PROTOCOL"}', ''); + +delete from dss_workflow_node where name = "sendemail"; +insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) +values('sendemail','sendemail','linkis.appconn.sendemail','0','0','1','1','0','icons/sendemail.icon'); + +select @sendemail_nodeId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.sendemail'; + +delete from `dss_workflow_node_to_group` where `node_id`=@sendemail_nodeId; + +delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@sendemail_nodeId; + +-- 查找节点所属组的id +select @sendemail_node_groupId:=id from `dss_workflow_node_group` where `name` = '数据输出'; + +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@sendemail_nodeId, @sendemail_node_groupId); + +-- 考虑表中有的是重复记录,最好加上limit 1 +select @sendemail_node_ui_lable_name_1:=id from `dss_workflow_node_ui` where `lable_name` = '节点名' limit 1; +select @sendemail_node_ui_lable_name_2:=id from `dss_workflow_node_ui` where `lable_name` = '节点描述' limit 1; +select @sendemail_node_ui_lable_name_3:=id from `dss_workflow_node_ui` where `lable_name` = '业务标签' limit 1; +select @sendemail_node_ui_lable_name_4:=id from `dss_workflow_node_ui` where `lable_name` = '应用标签' limit 1; +select @sendemail_node_ui_lable_name_5:=id from `dss_workflow_node_ui` where `lable_name` = '是否复用引擎' limit 1; +select @sendemail_node_ui_lable_name_6:=id from `dss_workflow_node_ui` where `lable_name` = '类型' limit 1; +select @sendemail_node_ui_lable_name_7:=id from `dss_workflow_node_ui` where `lable_name` = '邮件标题' limit 1; +select @sendemail_node_ui_lable_name_8:=id from `dss_workflow_node_ui` where `lable_name` = '收件人' limit 1; +select @sendemail_node_ui_lable_name_9:=id from `dss_workflow_node_ui` where `lable_name` = '抄送' limit 1; +select @sendemail_node_ui_lable_name_10:=id from `dss_workflow_node_ui` where `lable_name` = '秘密抄送' limit 1; +select @sendemail_node_ui_lable_name_11:=id from `dss_workflow_node_ui` where `lable_name` = '发送项' limit 1; + +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_4); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_5); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_6); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_7); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_8); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_9); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_10); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@sendemail_nodeId, @sendemail_node_ui_lable_name_11); diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/SendEmailRefExecutionOperation.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/SendEmailRefExecutionOperation.scala new file mode 100644 index 000000000..d4cae247a --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/SendEmailRefExecutionOperation.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail + +import java.util + +import com.webank.wedatasphere.dss.appconn.sendemail.conf.SendEmailAppConnInstanceConfiguration +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef.ExecutionResponseRefBuilder +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.{ExecutionResponseRef, RefExecutionRequestRef} +import com.webank.wedatasphere.dss.standard.app.development.operation.{AbstractDevelopmentOperation, RefExecutionOperation} +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef +import org.apache.linkis.common.utils.Utils + +import scala.collection.JavaConversions._ + +class SendEmailRefExecutionOperation + extends AbstractDevelopmentOperation[RefExecutionRequestRef.RefExecutionRequestRefImpl, ResponseRef] + with RefExecutionOperation[RefExecutionRequestRef.RefExecutionRequestRefImpl] { + + private val sendEmailAppConnHooks = SendEmailAppConnInstanceConfiguration.getSendEmailRefExecutionHooks + private val emailContentParsers = SendEmailAppConnInstanceConfiguration.getEmailContentParsers + private val emailContentGenerators = SendEmailAppConnInstanceConfiguration.getEmailContentGenerators + private val emailGenerator = SendEmailAppConnInstanceConfiguration.getEmailGenerator + private val emailSender = SendEmailAppConnInstanceConfiguration.getEmailSender + + + override def init(): Unit = { + super.init() + val properties = new util.HashMap[String, String] + service.getAppInstance.getConfig.foreach { + case (key: String, value: Object) if value != null => + properties.put(key, value.toString) + case _ => + } + emailSender.init(properties) + } + + override def execute(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl): ExecutionResponseRef = { + val email = Utils.tryCatch { + sendEmailAppConnHooks.foreach(_.preGenerate(requestRef)) + val email = emailGenerator.generateEmail(requestRef) + emailContentParsers.foreach{ + p => Utils.tryQuietly(p.parse(email)) + } + emailContentGenerators.foreach{ + g => Utils.tryQuietly(g.generate(email)) + } + sendEmailAppConnHooks.foreach(_.preSend(requestRef, email)) + email + }{ t => + return putErrorMsg("解析邮件内容失败!", t) + } + Utils.tryCatch { + emailSender.send(email) + new ExecutionResponseRefBuilder().success() + }(putErrorMsg("发送邮件失败!", _)) + } + + protected def putErrorMsg(errorMsg: String, t: Throwable): ExecutionResponseRef = { + logger.error(s"failed to send email, $errorMsg ", t) + new ExecutionResponseRefBuilder().setException(t).setErrorMsg(errorMsg).error() + } + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/conf/SendEmailAppConnConfiguration.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/conf/SendEmailAppConnConfiguration.scala new file mode 100644 index 000000000..cdd2f3728 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/conf/SendEmailAppConnConfiguration.scala @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.conf + +import org.apache.linkis.common.conf.CommonVars + +object SendEmailAppConnConfiguration { + + val EMAIL_SENDER_CLASS = CommonVars("wds.dss.appconn.email.sender.class", + "com.webank.wedatasphere.dss.appconn.sendemail.email.sender.EsbEmailSender") + + val EMAIL_HOOK_CLASSES = CommonVars("wds.dss.appconn.email.hook.classes", + "com.webank.wedatasphere.dss.appconn.sendemail.hook.SendEmailItsmCheckHook," + + "com.webank.wedatasphere.dss.appconn.sendemail.hook.SendEmailVisualisContentLimitHook," + + "com.webank.wedatasphere.dss.appconn.sendemail.hook.SendEmailTableauCheckHook") + + val EMAIL_IMAGE_HEIGHT = CommonVars("wds.dss.appconn.email.image.height", 500) + val EMAIL_IMAGE_WIDTH = CommonVars("wds.dss.appconn.email.image.width", 1920) + val DEFAULT_EMAIL_FROM = CommonVars("wds.dss.appconn.email.from.default", "") + val DEFAULT_EMAIL_SUFFIX = CommonVars("wds.dss.appconn.email.suffix.default", "@webank.com") + + val EMAIL_HOST = CommonVars("wds.dss.appconn.email.host", "") + val EMAIL_PORT: CommonVars[Integer] = CommonVars[Integer]("wds.dss.appconn.email.port", -1) + val EMAIL_PROTOCOL = CommonVars("wds.dss.appconn.email.protocol", "smtp") + val EMAIL_USERNAME = CommonVars("wds.dss.appconn.email.username", "") + val EMAIL_PASSWORD = CommonVars("wds.dss.appconn.email.password", "") + + val EMAIL_SMTP_AUTH = CommonVars("wds.dss.appconn.email.smtp.auth", "true") + val EMAIL_SMTP_STARTTLS_ENABLE = CommonVars("wds.dss.appconn.email.smtp.starttls.enable", "true") + val EMAIL_SMTP_STARTTLS_REQUIRED = CommonVars("wds.dss.appconn.email.smtp.starttls.required", "true") + val EMAIL_SMTP_SSL_ENABLED = CommonVars("wds.dss.appconn.email.smtp.ssl.enable", "true") + val EMAIL_SMTP_TIMEOUT: CommonVars[Integer] = CommonVars("wds.dss.appconn.email.smtp.timeout", 25000) + + val EMAIL_ENTITY_CLASSES = CommonVars("wds.dss.appconn.email.hook.entity.classes","com.webank.wedatasphere.dss.appconn.sendemail.hook.entity.newvisualis.NewVisualisEmailInfo," + + "com.webank.wedatasphere.dss.appconn.sendemail.hook.entity.visualis.VisualisEmailInfo") +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/cs/EmailCSHelper.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/cs/EmailCSHelper.scala new file mode 100644 index 000000000..674e133b0 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/cs/EmailCSHelper.scala @@ -0,0 +1,77 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.cs + + +import java.util + +import com.google.gson.internal.LinkedTreeMap +import com.webank.wedatasphere.dss.appconn.sendemail.exception.EmailSendFailedException +import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.cs.client.service.LinkisJobDataServiceImpl +import org.apache.linkis.cs.client.utils.{ContextServiceUtils, SerializeHelper} +import org.apache.linkis.cs.common.entity.enumeration.{ContextScope, ContextType} +import org.apache.linkis.cs.common.entity.source.CommonContextKey +import org.apache.linkis.cs.common.utils.CSCommonUtils +import org.apache.linkis.server.JSONUtils +import org.springframework.util.CollectionUtils + +import scala.collection.JavaConversions._ + + +object EmailCSHelper extends Logging{ + + /** + * update by peaceWong form cs to get job ID + */ + def getJobIds(refContext: ExecutionRequestRefContext): Array[Long] = { + val contextIDStr = ContextServiceUtils.getContextIDStrByMap(refContext.getRuntimeMap) + val nodeIDs = refContext.getRuntimeMap.get("content") match { + case string: String => JSONUtils.gson.fromJson(string, classOf[java.util.List[String]]) + case list: java.util.List[String] => list + } + if (null == nodeIDs || nodeIDs.length < 1){ + throw new EmailSendFailedException(80003 ,"empty result set is not allowed") + } + info(s"From cs to getJob ids $nodeIDs.") + val jobIds = nodeIDs.map(ContextServiceUtils.getNodeNameByNodeID(contextIDStr, _)).map{ nodeName => + val contextKey = new CommonContextKey + contextKey.setContextScope(ContextScope.PUBLIC) + contextKey.setContextType(ContextType.DATA) + contextKey.setKey(CSCommonUtils.NODE_PREFIX + nodeName + CSCommonUtils.JOB_ID) + LinkisJobDataServiceImpl.getInstance().getLinkisJobData(contextIDStr, SerializeHelper.serializeContextKey(contextKey)) + }.map(_.getJobID).toArray + if (null == jobIds || jobIds.length < 1){ + throw new EmailSendFailedException(80003 ,"empty result set is not allowed") + } + info(s"Job IDs is ${jobIds.toList}.") + jobIds + } + + def getJobTypes(refContext: ExecutionRequestRefContext):util.ArrayList[String] = { + val jobTypes = new util.ArrayList[String]() + getJobIds(refContext).foreach { jobId => + jobTypes.add(refContext.fetchLinkisJob(jobId).getParams.get("labels").asInstanceOf[LinkedTreeMap[_,_]].get("codeType").toString) + } + if (CollectionUtils.isEmpty(jobTypes)) { + throw new EmailSendFailedException(80003 ,"empty result set is not allowed") + } + info(s"Job Types is $jobTypes.") + jobTypes + } +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/EmailGenerator.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/EmailGenerator.scala new file mode 100644 index 000000000..32c24529e --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/EmailGenerator.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email + +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef + +trait EmailGenerator { + + def generateEmail(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl): Email + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/EmailSender.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/EmailSender.scala new file mode 100644 index 000000000..7531466e9 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/EmailSender.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email + +import java.util +import java.util.concurrent.Future + +import com.webank.wedatasphere.dss.appconn.sendemail.exception.EmailSendFailedException + +trait EmailSender { + + def init(properties: util.Map[String, String]): Unit + + @throws(classOf[EmailSendFailedException]) + def send(email: Email): Unit + + def sendAsync(email: Email): Future[Unit] + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/AbstractEmail.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/AbstractEmail.scala new file mode 100644 index 000000000..3ef44dfe3 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/AbstractEmail.scala @@ -0,0 +1,59 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email.domain + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email + +import scala.collection.mutable.ArrayBuffer + +class AbstractEmail extends Email { + + private var content: String = _ + private var attachments = ArrayBuffer[Attachment]() + private var subject: String = _ + private var from: String = _ + private var to: String = _ + private var cc: String = _ + private var bcc: String = _ + private var emialType: String = _ + + override def getContent: String = content + override def setContent(content: String): Unit = this.content = content + + override def getAttachments: Array[Attachment] = attachments.toArray + override def setAttachments(attachments: Array[Attachment]): Unit = + this.attachments ++= attachments + def addAttachment(attachment: Attachment): Unit = this.attachments += attachment + + override def getSubject: String = subject + override def setSubject(subject: String): Unit = this.subject = subject + + override def getFrom: String = from + override def setFrom(from: String): Unit = this.from = from + + override def getTo: String = to + override def setTo(to: String): Unit = this.to = to + + override def getCc: String = cc + override def setCc(cc: String): Unit = this.cc = cc + + override def getBcc: String = bcc + override def setBcc(bcc: String): Unit = this.bcc = bcc + + override def getEmailType: String = emialType + override def setEmailType(emailType: String): Unit = this.emialType = emailType +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/Attachment.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/Attachment.scala new file mode 100644 index 000000000..dbea0da83 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/Attachment.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email.domain + +import java.io.File + +trait Attachment { + def getName: String + def getBase64Str: String + def getFile: File + def getMediaType: String +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/MultiContentEmail.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/MultiContentEmail.scala new file mode 100644 index 000000000..086488106 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/MultiContentEmail.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email.domain + +import java.util + +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.EmailContent + +import scala.collection.JavaConversions._ + +class MultiContentEmail extends AbstractEmail { + + private val emailContents = new util.ArrayList[EmailContent[_]]() + + def addEmailContent(emailContent: EmailContent[_]): Unit = emailContents.add(emailContent) + + def getEmailContents: Array[EmailContent[_]] = emailContents.toIterator.toArray + + + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/PngAttachment.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/PngAttachment.scala new file mode 100644 index 000000000..369a55cb0 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/PngAttachment.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email.domain + +import java.io.File + +class PngAttachment(name: String, b64: String) extends Attachment { + + override def getName: String = name + + override def getBase64Str: String = b64 + + override def getFile: File = null //TODO write b64 to file + + override def getMediaType: String = "image/png" + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/generate/AbstractEmailGenerator.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/generate/AbstractEmailGenerator.scala new file mode 100644 index 000000000..556303371 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/generate/AbstractEmailGenerator.scala @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email.generate + +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.AbstractEmail +import com.webank.wedatasphere.dss.appconn.sendemail.email.{Email, EmailGenerator} +import com.webank.wedatasphere.dss.common.utils.VariableUtils +import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef +import org.apache.linkis.common.utils.Logging + +trait AbstractEmailGenerator extends EmailGenerator with Logging{ + + protected def createEmail(): AbstractEmail + + override def generateEmail(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl): Email = { + val email = createEmail() + generateEmailInfo(requestRef, email) + generateEmailContent(requestRef, email) + email + } + + protected def getRuntimeMap(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl): java.util.Map[String, AnyRef] = + requestRef.getExecutionRequestRefContext.getRuntimeMap + + protected def getExecutionRequestRefContext(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl): ExecutionRequestRefContext = + requestRef.getExecutionRequestRefContext + + protected def generateEmailInfo(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl, email: AbstractEmail): Unit = { + import scala.collection.JavaConversions._ + val runtimeMap = getRuntimeMap(requestRef) + runtimeMap foreach { + case (k, v) => logger.info(s"K is $k, V is $v") + } + val subject = if (runtimeMap.get("subject") != null) { + VariableUtils.replace(runtimeMap.get("subject").toString) + } else{ + "This is an email" + } + email.setSubject(subject) + val bcc = if (runtimeMap.get("bcc") != null) runtimeMap.get("bcc").toString else "" + email.setBcc(bcc) + val cc = if (runtimeMap.get("cc") != null) runtimeMap.get("cc").toString else "" + email.setCc(cc) + val from = if (runtimeMap.get("from") != null) runtimeMap.get("from").toString else + if(runtimeMap.get("wds.dss.workflow.submit.user") != null){ + runtimeMap.get("wds.dss.workflow.submit.user").toString + } else runtimeMap.get("user").toString + email.setFrom(from) + val to = if (runtimeMap.get("to") != null) runtimeMap.get("to").toString else "" + email.setTo(to) + } + + protected def generateEmailContent(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl, email: AbstractEmail): Unit + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/generate/MultiContentEmailGenerator.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/generate/MultiContentEmailGenerator.scala new file mode 100644 index 000000000..7043d54d5 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/generate/MultiContentEmailGenerator.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.email.generate + +import com.google.gson.internal.LinkedTreeMap +import com.webank.wedatasphere.dss.appconn.sendemail.cs.EmailCSHelper +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.{AbstractEmail, MultiContentEmail} +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain.PictureEmailContent +import com.webank.wedatasphere.dss.appconn.sendemail.exception.EmailSendFailedException +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.storage.resultset.ResultSetFactory + +class MultiContentEmailGenerator extends AbstractEmailGenerator { + + override protected def createEmail(): AbstractEmail = new MultiContentEmail + + override protected def generateEmailContent(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl, email: AbstractEmail): Unit = email match { + case multiContentEmail: MultiContentEmail => + val runtimeMap = getRuntimeMap(requestRef) + val refContext = getExecutionRequestRefContext(requestRef) + runtimeMap.get("category") match { + case "node" => + val resultSetFactory = ResultSetFactory.getInstance + EmailCSHelper.getJobIds(refContext).foreach { jobId => + refContext.fetchLinkisJobResultSetPaths(jobId).foreach { fsPath => + val resultSet = resultSetFactory.getResultSetByPath(fsPath) + val emailContent = resultSet.resultSetType() match { + case ResultSetFactory.PICTURE_TYPE => new PictureEmailContent(fsPath) + case ResultSetFactory.HTML_TYPE => throw new EmailSendFailedException(80003 ,"html result set is not allowed")//new HtmlEmailContent(fsPath) + case ResultSetFactory.TABLE_TYPE => throw new EmailSendFailedException(80003 ,"table result set is not allowed")//new TableEmailContent(fsPath) + case ResultSetFactory.TEXT_TYPE => throw new EmailSendFailedException(80003 ,"text result set is not allowed")//new FileEmailContent(fsPath) + } + multiContentEmail.addEmailContent(emailContent) + } + if (StringUtils.isBlank(multiContentEmail.getEmailType)) { + val emailType = refContext.fetchLinkisJob(jobId).getParams.get("labels").asInstanceOf[LinkedTreeMap[_,_]].get("codeType").toString + multiContentEmail.setEmailType(emailType) + } + } + case "file" => throw new EmailSendFailedException(80003 ,"file content is not allowed") //addContentEmail(c => new FileEmailContent(new FsPath(c))) + case "text" => throw new EmailSendFailedException(80003 ,"text content is not allowed")//addContentEmail(new TextEmailContent(_)) + case "link" => throw new EmailSendFailedException(80003 ,"link content is not allowed")//addContentEmail(new UrlEmailContent(_)) + } + } + + + + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContentGenerator.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContentGenerator.scala new file mode 100644 index 000000000..a6235abd2 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContentGenerator.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email + +trait EmailContentGenerator { + + def generate(email: Email): Unit + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContentParser.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContentParser.scala new file mode 100644 index 000000000..a85d52269 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/EmailContentParser.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email + +trait EmailContentParser { + + def parse(email: Email): Unit + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/ArrayEmailContent.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/ArrayEmailContent.scala new file mode 100644 index 000000000..2fc552dd8 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/ArrayEmailContent.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain + +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.EmailContent + +class ArrayEmailContent extends EmailContent[Array[String]] { + + private var content: Array[String] = _ + + override def getContent: Array[String] = content + + override def setContent(content: Array[String]): Unit = this.content = content + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/FsPathStoreEmailContent.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/FsPathStoreEmailContent.scala new file mode 100644 index 000000000..ebbe876fc --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/FsPathStoreEmailContent.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain + +import org.apache.linkis.common.io.FsPath + +trait FsPathStoreEmailContent { + private var fsPath: FsPath = _ + + def getFsPath: FsPath = fsPath + def setFsPath(fsPath: FsPath): Unit = this.fsPath = fsPath +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/StringEmailContent.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/StringEmailContent.scala new file mode 100644 index 000000000..e28ab0bc8 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/StringEmailContent.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain + +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.EmailContent + +class StringEmailContent extends EmailContent[String] { + + private var content: String = _ + + override def getContent: String = content + + override def setContent(content: String): Unit = this.content = content + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/package.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/package.scala new file mode 100644 index 000000000..016398556 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/domain/package.scala @@ -0,0 +1,81 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain + +import org.apache.linkis.common.io.FsPath + +import scala.beans.BeanProperty + +package object emailcontent { + +} + +class PictureEmailContent extends ArrayEmailContent with FsPathStoreEmailContent { + + def this(filePath: FsPath) = { + this() + setFsPath(filePath) + } + +} + +class HtmlEmailContent extends StringEmailContent with FsPathStoreEmailContent { + + def this(filePath: FsPath) = { + this() + setFsPath(filePath) + } + +} + +class TableEmailContent extends StringEmailContent with FsPathStoreEmailContent { + + def this(filePath: FsPath) = { + this() + setFsPath(filePath) + } + +} + +class FileEmailContent extends StringEmailContent with FsPathStoreEmailContent { + + def this(filePath: FsPath) = { + this() + setFsPath(filePath) + } + +} + +class TextEmailContent extends StringEmailContent { + + def this(text: String) = { + this() + setContent(text) + } + +} + +class UrlEmailContent extends StringEmailContent { + + @BeanProperty var url: String = _ + + def this(url: String) = { + this() + this.url = url + } + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/generator/AbstractEmailContentGenerator.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/generator/AbstractEmailContentGenerator.scala new file mode 100644 index 000000000..d2b1e99f6 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/generator/AbstractEmailContentGenerator.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.generator + +import java.text.SimpleDateFormat +import java.util.{Calendar, Date} + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.EmailContentGenerator + + +trait AbstractEmailContentGenerator extends EmailContentGenerator { + + protected def formatSubjectOfOldVersion(email: Email): Unit = { + var title = email.getSubject + if (title.contains("YYYY-MM-DD HH:MM:SS")) { + val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + val timeStr = sdf.format(new Date) + title = title.replace("YYYY-MM-DD HH:MM:SS", timeStr) + } else if (title.contains("YYYY-MM-DD-1")) { + val sdf = new SimpleDateFormat("yyyy-MM-dd") + val calendar = Calendar.getInstance + calendar.add(Calendar.DATE, -1) + val timeStr = sdf.format(calendar.getTime) + title = title.replace("YYYY-MM-DD-1", timeStr) + } else if (title.contains("YYYY-MM-DD")) { + val sdf = new SimpleDateFormat("yyyy-MM-dd") + val timeStr = sdf.format(new Date) + title = title.replace("YYYY-MM-DD", timeStr) + } else { + if (title.contains("NO_TIMESTAMP")) { + title = title.replaceAll("NO_TIMESTAMP", "") + } else { + val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + val timeStr = sdf.format(new Date) + title = title + timeStr + } + } + email.setSubject(title) + } + + protected def formatSubject(email: Email): Unit = { + + } + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/generator/MultiEmailContentGenerator.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/generator/MultiEmailContentGenerator.scala new file mode 100644 index 000000000..af40d314c --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/generator/MultiEmailContentGenerator.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.generator + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.MultiContentEmail +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain.{ArrayEmailContent, StringEmailContent} +import org.apache.linkis.common.utils.Logging + + +class MultiEmailContentGenerator extends AbstractEmailContentGenerator with Logging { + + override def generate(email: Email): Unit = email match { + case multiContentEmail: MultiContentEmail => + formatSubjectOfOldVersion(email) + formatSubject(multiContentEmail) + formatContent(multiContentEmail) + } + + protected def formatContent(email: MultiContentEmail): Unit = { + val sb: StringBuilder = new StringBuilder("") + sb.append("") + email.getEmailContents.foreach { + case emailContent: ArrayEmailContent => + emailContent.getContent.foreach(content => sb.append("")) + case emailContent: StringEmailContent => + sb.append("") + } + sb.append("
").append(content).append("
").append(emailContent.getContent).append("
") + sb.append("") + email.setContent(sb.toString) + } + +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/AbstractEmailContentParser.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/AbstractEmailContentParser.scala new file mode 100644 index 000000000..2bbcb6c21 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/AbstractEmailContentParser.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser + +import java.lang.reflect.{ParameterizedType, Type} + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.MultiContentEmail +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain.FsPathStoreEmailContent +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.{EmailContent, EmailContentParser} +import org.apache.linkis.common.io.resultset.ResultSetReader +import org.apache.linkis.common.io.{MetaData, Record} +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.storage.LineRecord +import org.apache.linkis.storage.resultset.ResultSetReader +import org.apache.commons.io.IOUtils + +abstract class AbstractEmailContentParser[T] extends EmailContentParser { + + override def parse(email: Email): Unit = email match { + case multiContentEmail: MultiContentEmail => + multiContentEmail.getEmailContents.foreach { + case t: EmailContent[_] if t.getClass == getEmailContentClass => + parseEmailContent(t.asInstanceOf[T], multiContentEmail) + case _ => + } + case _ => + } + + protected def getResultSetReader(fsPathStore: FsPathStoreEmailContent): ResultSetReader[_ <: MetaData, _ <: Record] = { + val reader = ResultSetReader.getResultSetReader(fsPathStore.getFsPath.getSchemaPath) + reader.getMetaData + reader + } + + protected def getFirstLineRecord(fsPathStore: FsPathStoreEmailContent): Option[String] = { + val reader = getResultSetReader(fsPathStore) + if(!reader.hasNext) None + else Utils.tryFinally(reader.getRecord match { + case record: LineRecord => Option(record.getLine) + })(IOUtils.closeQuietly(reader)) + } + + protected def getEmailContentClass: Type = getClass.getGenericSuperclass match { + case pType: ParameterizedType => pType.getActualTypeArguments.head + } + + protected def parseEmailContent(emailContent: T, + multiContentEmail: MultiContentEmail): Unit + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/FileEmailContentParser.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/FileEmailContentParser.scala new file mode 100644 index 000000000..a6aa2daf9 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/FileEmailContentParser.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser + +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.MultiContentEmail +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain.FileEmailContent +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.storage.LineRecord +import org.apache.commons.io.IOUtils + +object FileEmailContentParser extends AbstractEmailContentParser[FileEmailContent] { + override protected def parseEmailContent(emailContent: FileEmailContent, + multiContentEmail: MultiContentEmail): Unit = { + val reader = getResultSetReader(emailContent) + val content = new StringBuilder + Utils.tryFinally{ + while(reader.hasNext) { + reader.getRecord match { + case lineRecord: LineRecord => + content.append(lineRecord.getLine).append("
") + } + } + }(IOUtils.closeQuietly(reader)) + emailContent.setContent(content.toString()) + } +} \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/HtmlEmailContentParser.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/HtmlEmailContentParser.scala new file mode 100644 index 000000000..cab083f70 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/HtmlEmailContentParser.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser + +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.MultiContentEmail +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain.HtmlEmailContent + +object HtmlEmailContentParser extends AbstractEmailContentParser[HtmlEmailContent] { + override protected def parseEmailContent(emailContent: HtmlEmailContent, + multiContentEmail: MultiContentEmail): Unit = { + getFirstLineRecord(emailContent).foreach(emailContent.setContent) + } +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/PictureEmailContentParser.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/PictureEmailContentParser.scala new file mode 100644 index 000000000..516248608 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/PictureEmailContentParser.scala @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser + +import java.awt.image.BufferedImage +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import java.util.{Base64, UUID} + +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.{AbstractEmail, MultiContentEmail, PngAttachment} +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain.PictureEmailContent +import org.apache.linkis.common.conf.Configuration +import javax.imageio.ImageIO +import org.apache.commons.codec.binary.Base64OutputStream +import com.webank.wedatasphere.dss.appconn.sendemail.conf.SendEmailAppConnConfiguration._ + +object PictureEmailContentParser extends AbstractEmailContentParser[PictureEmailContent] { + + override protected def parseEmailContent(emailContent: PictureEmailContent, + multiContentEmail: MultiContentEmail): Unit = { + getFirstLineRecord(emailContent).foreach { imageStr => + val decoder = Base64.getDecoder + val byteArr = decoder.decode(imageStr) + val inputStream = new ByteArrayInputStream(byteArr) + val image = ImageIO.read(inputStream) + val contents = generateImage(image, multiContentEmail) + emailContent.setContent(contents) + } + } + + protected def generateImage(bufferedImage: BufferedImage, email: AbstractEmail): Array[String] = { + val imageUUID: String = UUID.randomUUID.toString + val width: Int = bufferedImage.getWidth + val height: Int = bufferedImage.getHeight + // 只支持修改visualis图片大小,后续如果有新增其他类型的邮件需要修改图片大小,需要在if中加上该邮件类型 + val imagesCuts = if (email.getEmailType.contains("visualis") && height > EMAIL_IMAGE_HEIGHT.getValue) { + val numOfCut = Math.ceil(height.toDouble / EMAIL_IMAGE_HEIGHT.getValue).toInt + val realHeight = height / numOfCut + (0 until numOfCut).map(i => bufferedImage.getSubimage(0, i * realHeight, width, realHeight)).toArray + } else Array(bufferedImage) + imagesCuts.indices.map { index => + val image = imagesCuts(index) + val imageName = index + "_" + imageUUID + ".png" + val os = new ByteArrayOutputStream + val b64Stream = new Base64OutputStream(os) + ImageIO.write(image, "png", b64Stream) + val b64 = os.toString(Configuration.BDP_ENCODING.getValue) + email.addAttachment(new PngAttachment(imageName, b64)) + + var iHeight = image.getHeight + var iWidth = image.getWidth + + if (email.getEmailType.contains("visualis") && iWidth > EMAIL_IMAGE_WIDTH.getValue) { + iHeight = ((EMAIL_IMAGE_WIDTH.getValue.toDouble / iWidth.toDouble) * iHeight.toDouble).toInt + iWidth = EMAIL_IMAGE_WIDTH.getValue + } + s"""""" + }.toArray + } + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/TableEmailContentParser.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/TableEmailContentParser.scala new file mode 100644 index 000000000..71dbd7709 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/TableEmailContentParser.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.parser + +import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.MultiContentEmail +import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain.TableEmailContent +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} +import org.apache.commons.io.IOUtils +import org.apache.commons.lang.StringUtils + +object TableEmailContentParser extends AbstractEmailContentParser[TableEmailContent] { + override protected def parseEmailContent(emailContent: TableEmailContent, + multiContentEmail: MultiContentEmail): Unit = { + val reader = getResultSetReader(emailContent) + val content = new StringBuilder + reader.getMetaData match { + case tableMetaData: TableMetaData => + writeTableTH(tableMetaData, content) + } + Utils.tryFinally { + while(reader.hasNext) { + reader.getRecord match { + case tableRecord: TableRecord => + writeTableTR(tableRecord, content) + } + } + }(IOUtils.closeQuietly(reader)) + emailContent.setContent(content.toString()) + } + + protected def writeTableTH(tableMetaData: TableMetaData, content: StringBuilder): Unit = { + content.append("") + tableMetaData.columns.foreach{ c => + content.append("").append(c.columnName) + if(StringUtils.isNotBlank(c.comment)) { + val comment = if(c.comment.length < 10) c.comment else c.comment.substring(0, 9) + "..." + content.append("(").append(comment).append(")") + } + content.append("") + } + content.append("") + } + + protected def writeTableTR(tableRecord: TableRecord, content: StringBuilder): Unit = { + content.append("") + tableRecord.row.foreach(v => content.append("").append(v).append("")) + content.append("") + } + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/hook/AbstractSendEmailRefExecutionHook.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/hook/AbstractSendEmailRefExecutionHook.scala new file mode 100644 index 000000000..294e385f5 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/hook/AbstractSendEmailRefExecutionHook.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.hook + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email +import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef + +abstract class AbstractSendEmailRefExecutionHook extends SendEmailRefExecutionHook { + + protected def getExecutionRequestRefContext(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl): ExecutionRequestRefContext = + requestRef.getExecutionRequestRefContext + + override def preSend(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl, email: Email): Unit = {} +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/hook/SendEmailRefExecutionHook.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/hook/SendEmailRefExecutionHook.scala new file mode 100644 index 000000000..37838fe49 --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/hook/SendEmailRefExecutionHook.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.sendemail.hook + +import com.webank.wedatasphere.dss.appconn.sendemail.email.Email +import com.webank.wedatasphere.dss.appconn.sendemail.exception.EmailSendFailedException +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef + + +trait SendEmailRefExecutionHook { + + @throws(classOf[EmailSendFailedException]) + def preGenerate(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl): Unit + + def preSend(requestRef: RefExecutionRequestRef.RefExecutionRequestRefImpl, email: Email): Unit + +} diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/org/apache/linkis/server/JSONUtils.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/org/apache/linkis/server/JSONUtils.scala new file mode 100644 index 000000000..4474122fc --- /dev/null +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/org/apache/linkis/server/JSONUtils.scala @@ -0,0 +1,14 @@ +package org.apache.linkis.server + +/** + * + * @date 2022-03-11 + * @author enjoyyin + * @since 0.5.0 + */ +object JSONUtils { + + val gson = BDPJettyServerHelper.gson + + val jackson = BDPJettyServerHelper.jacksonJson +} diff --git a/dss-appconn/appconns/dss-sso-appconn/pom.xml b/dss-appconn/appconns/dss-sso-appconn/pom.xml new file mode 100644 index 000000000..e17d4ae5f --- /dev/null +++ b/dss-appconn/appconns/dss-sso-appconn/pom.xml @@ -0,0 +1,106 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-sso-appconn + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + provided + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-sender-service + ${dss.version} + provided + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-sso-appconn/src/main/assembly/distribution.xml b/dss-appconn/appconns/dss-sso-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..c186e12f6 --- /dev/null +++ b/dss-appconn/appconns/dss-sso-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,74 @@ + + + + dss-sso-appconn + + dir + + true + sso + + + + + + lib + true + true + false + true + true + + + + + + + + ${basedir}/conf + + * + + 0777 + conf + unix + + + . + + */** + + logs + + + + ${basedir}/src/main/resources + + init.sql + + 0777 + db + + + + + + + diff --git a/dss-appconn/appconns/dss-sso-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/sso/SSOAppConn.java b/dss-appconn/appconns/dss-sso-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/sso/SSOAppConn.java new file mode 100644 index 000000000..22500e945 --- /dev/null +++ b/dss-appconn/appconns/dss-sso-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/sso/SSOAppConn.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.appconn.sso; + +import com.webank.wedatasphere.dss.appconn.core.impl.AbstractOnlySSOAppConn; + +/** + * @author enjoyyin + * @date 2022-03-31 + * @since 1.1.0 + */ +public class SSOAppConn extends AbstractOnlySSOAppConn { + @Override + protected void initialize() { + if(getAppDesc() != null) { + logger.info("Load a SSOAppConn " + getAppDesc().getAppName()); + } + } +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/pom.xml b/dss-appconn/appconns/dss-workflow-appconn/pom.xml new file mode 100644 index 000000000..937ba44ee --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/pom.xml @@ -0,0 +1,112 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-workflow-appconn + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + provided + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-sender-service + ${dss.version} + provided + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + \ No newline at end of file diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/assembly/distribution.xml b/dss-appconn/appconns/dss-workflow-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..003e51313 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,66 @@ + + + + dss-workflow-appconn + + dir + + true + workflow + + + + + + lib + true + true + false + true + true + + + + + + ${basedir}/src/main/resources + + appconn.properties + + 0777 + / + unix + + + + ${basedir}/src/main/resources + + log4j.properties + log4j2.xml + + 0777 + conf + unix + + + + + + diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/WorkflowAppConn.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/WorkflowAppConn.java new file mode 100644 index 000000000..964c38831 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/WorkflowAppConn.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow; + +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.appconn.core.impl.AbstractAppConn; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; + +public class WorkflowAppConn extends AbstractAppConn implements OnlyDevelopmentAppConn { + + private DevelopmentIntegrationStandard developmentIntegrationStandard = new WorkflowDevelopmentIntegrationStandard(); + + @Override + protected void initialize() { + } + + @Override + public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() { + return developmentIntegrationStandard; + } +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/WorkflowDevelopmentIntegrationStandard.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/WorkflowDevelopmentIntegrationStandard.java new file mode 100644 index 000000000..f93d4c396 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/WorkflowDevelopmentIntegrationStandard.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow; + +import com.webank.wedatasphere.dss.appconn.workflow.service.WorkflowCRUDService; +import com.webank.wedatasphere.dss.appconn.workflow.service.WorkflowExportService; +import com.webank.wedatasphere.dss.appconn.workflow.service.WorkflowImportService; +import com.webank.wedatasphere.dss.appconn.workflow.service.WorkflowQueryService; +import com.webank.wedatasphere.dss.standard.app.development.service.*; +import com.webank.wedatasphere.dss.standard.app.development.standard.AbstractDevelopmentIntegrationStandard; + +public class WorkflowDevelopmentIntegrationStandard extends AbstractDevelopmentIntegrationStandard { + + @Override + protected RefCRUDService createRefCRUDService() { + return new WorkflowCRUDService(); + } + + @Override + protected RefExecutionService createRefExecutionService() { + return null; + } + + @Override + protected RefExportService createRefExportService() { + return new WorkflowExportService(); + } + + @Override + protected RefImportService createRefImportService() { + return new WorkflowImportService(); + } + + @Override + protected RefQueryService createRefQueryService() { + return new WorkflowQueryService(); + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java new file mode 100644 index 000000000..28d8a9781 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.opertion; + +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCopyOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.ThirdlyRequestRef; +import com.webank.wedatasphere.dss.workflow.common.protocol.RequestCopyWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseCopyWorkflow; +import org.apache.linkis.rpc.Sender; + +import java.util.HashMap; +import java.util.Map; + + +public class WorkflowRefCopyOperation + extends AbstractDevelopmentOperation + implements RefCopyOperation { + + private final Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(); + + @Override + public RefJobContentResponseRef copyRef(ThirdlyRequestRef.CopyWitContextRequestRefImpl workflowCopyRequestRef) { + Long appId = (Long) workflowCopyRequestRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY); + String userName = workflowCopyRequestRef.getUserName(); + String contextIdStr = workflowCopyRequestRef.getContextId(); + String projectName = workflowCopyRequestRef.getProjectName(); + //插入version + String version = workflowCopyRequestRef.getNewVersion(); + String description = (String) workflowCopyRequestRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_DESCRIPTION); + RequestCopyWorkflow requestCopyWorkflow = new RequestCopyWorkflow(userName, + workflowCopyRequestRef.getWorkspace(), appId, contextIdStr, + projectName, version, description, workflowCopyRequestRef.getDSSLabels()); + ResponseCopyWorkflow responseCopyWorkflow = (ResponseCopyWorkflow) sender.ask(requestCopyWorkflow); + Map refJobContent = new HashMap<>(2); + refJobContent.put(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, responseCopyWorkflow.getDssFlow().getId()); + refJobContent.put(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY, responseCopyWorkflow.getDssFlow().getFlowJson()); + return RefJobContentResponseRef.newBuilder().setRefJobContent(refJobContent).success(); + } +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCreationOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCreationOperation.java new file mode 100644 index 000000000..9fcbc1407 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCreationOperation.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.opertion; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCreationOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.ThirdlyRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.workflow.common.protocol.RequestCreateWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseCreateWorkflow; +import org.apache.linkis.rpc.Sender; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class WorkflowRefCreationOperation + extends AbstractDevelopmentOperation + implements RefCreationOperation { + + private Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(); + + @Override + public RefJobContentResponseRef createRef(ThirdlyRequestRef.DSSJobContentWithContextRequestRef requestRef) throws ExternalOperationFailedException { + //发送RPC请求 + DSSOrchestratorInfo dssOrchestratorInfo = (DSSOrchestratorInfo) requestRef.getDSSJobContent().get(OrchestratorRefConstant.DSS_ORCHESTRATOR_INFO_KEY); + String userName = requestRef.getUserName(); + String workflowName = dssOrchestratorInfo.getName(); + String contextId = requestRef.getContextId() != null ? requestRef.getContextId() : ""; + String description = dssOrchestratorInfo.getDesc(); + List dssLabels = requestRef.getDSSLabels(); + String orcVersion = (String) requestRef.getDSSJobContent().get(OrchestratorRefConstant.ORCHESTRATOR_VERSION_KEY); + String schedulerAppConnName = (String) requestRef.getDSSJobContent().get(OrchestratorRefConstant.ORCHESTRATION_SCHEDULER_APP_CONN); + long parentFlowId = -1L; + List linkedAppConnNames = dssOrchestratorInfo.getLinkedAppConnNames() != null ? + dssOrchestratorInfo.getLinkedAppConnNames() : new ArrayList<>(); + String uses = dssOrchestratorInfo.getUses() != null ? + dssOrchestratorInfo.getUses() : "uses"; + RequestCreateWorkflow requestCreateWorkflow = new RequestCreateWorkflow(userName, dssOrchestratorInfo.getProjectId(), workflowName, + contextId, description, parentFlowId, uses, linkedAppConnNames, dssLabels, orcVersion, schedulerAppConnName); + + ResponseCreateWorkflow responseCreateWorkflow = (ResponseCreateWorkflow) sender.ask(requestCreateWorkflow); + Map refJobContent = MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, responseCreateWorkflow.getDssFlow().getId(), + OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY, responseCreateWorkflow.getDssFlow().getFlowJson()); + return RefJobContentResponseRef.newBuilder().setRefJobContent(refJobContent).success(); + } +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefDeletionOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefDeletionOperation.java new file mode 100644 index 000000000..fbf6fd365 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefDeletionOperation.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.opertion; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.protocol.JobStatus; +import com.webank.wedatasphere.dss.common.protocol.RequestDeleteWorkflow; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.OnlyDevelopmentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseDeleteWorkflow; +import org.apache.linkis.rpc.Sender; + +import java.util.List; + + +public class WorkflowRefDeletionOperation + extends AbstractDevelopmentOperation + implements RefDeletionOperation { + + @Override + public ResponseRef deleteRef(OnlyDevelopmentRequestRef.RefJobContentRequestRefImpl requestRef) throws ExternalOperationFailedException { + String userName = requestRef.getUserName(); + Long flowId = (Long) requestRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY); + RequestDeleteWorkflow requestDeleteWorkflow = new RequestDeleteWorkflow(userName, flowId); + List dssLabels = requestRef.getDSSLabels(); + Sender tempSend = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(dssLabels); + ResponseDeleteWorkflow responseDeleteWorkflow = (ResponseDeleteWorkflow) tempSend.ask(requestDeleteWorkflow); + if(responseDeleteWorkflow.getJobStatus() == JobStatus.Success) { + return ResponseRef.newInternalBuilder().success(); + } else { + return ResponseRef.newInternalBuilder().error("Unknown error, please ask admin for help."); + } + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java new file mode 100644 index 000000000..9b1c5c12c --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.opertion; + +import com.webank.wedatasphere.dss.common.protocol.RequestExportWorkflow; +import com.webank.wedatasphere.dss.common.protocol.ResponseExportWorkflow; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ExportResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.ThirdlyRequestRef; +import org.apache.linkis.rpc.Sender; + +import java.util.HashMap; +import java.util.Map; + + +public class WorkflowRefExportOperation + extends AbstractDevelopmentOperation + implements RefExportOperation { + + @Override + public ExportResponseRef exportRef(ThirdlyRequestRef.RefJobContentRequestRefImpl requestRef) { + + String userName = requestRef.getUserName(); + long flowId = (long) requestRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY); + Long projectId = requestRef.getRefProjectId(); + String projectName = requestRef.getProjectName(); + RequestExportWorkflow requestExportWorkflow = new RequestExportWorkflow(userName, + flowId, + projectId, + projectName, + toJson(requestRef.getWorkspace()), + requestRef.getDSSLabels()); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(requestRef.getDSSLabels()); + ResponseExportWorkflow responseExportWorkflow = (ResponseExportWorkflow) sender.ask(requestExportWorkflow); + Map resourceMap = new HashMap<>(2); + resourceMap.put(ImportRequestRef.RESOURCE_ID_KEY, responseExportWorkflow.resourceId()); + resourceMap.put(ImportRequestRef.RESOURCE_VERSION_KEY, responseExportWorkflow.version()); + return ExportResponseRef.newBuilder().setResourceMap(resourceMap).success(); + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefImportOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefImportOperation.java new file mode 100644 index 000000000..19c2172bf --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefImportOperation.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.opertion; + +import com.webank.wedatasphere.dss.common.protocol.JobStatus; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.ThirdlyRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.workflow.common.protocol.RequestImportWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseImportWorkflow; +import org.apache.linkis.rpc.Sender; + +import java.util.HashMap; +import java.util.Map; + +public class WorkflowRefImportOperation + extends AbstractDevelopmentOperation + implements RefImportOperation { + + @Override + public RefJobContentResponseRef importRef(ThirdlyRequestRef.ImportWitContextRequestRefImpl requestRef) throws ExternalOperationFailedException { + RequestImportWorkflow requestImportWorkflow = new RequestImportWorkflow(requestRef.getUserName(), + (String) requestRef.getResourceMap().get(ImportRequestRef.RESOURCE_ID_KEY), + (String) requestRef.getResourceMap().get(ImportRequestRef.RESOURCE_VERSION_KEY), + requestRef.getRefProjectId(), requestRef.getProjectName(), + requestRef.getNewVersion(), + requestRef.getWorkspace(), + requestRef.getContextId(), requestRef.getDSSLabels()); + + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(requestRef.getDSSLabels()); + ResponseImportWorkflow responseImportWorkflow = (ResponseImportWorkflow) sender.ask(requestImportWorkflow); + if(responseImportWorkflow.getStatus() == JobStatus.Success) { + if(MapUtils.isEmpty(responseImportWorkflow.getWorkflows())) { + return RefJobContentResponseRef.newBuilder() + .error("Empty workflow returned from workflow server, please ask admin for help!"); + } + Map refMap = new HashMap<>(2); + responseImportWorkflow.getWorkflows().forEach((key, value) -> { + refMap.put(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, key); + refMap.put(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY, value); + }); + return RefJobContentResponseRef.newBuilder().setRefJobContent(refMap).success(); + } else { + return RefJobContentResponseRef.newBuilder().error("Unknown reason, please ask admin for help."); + } + } +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefQueryOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefQueryOperation.java new file mode 100644 index 000000000..98f5f1ca5 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefQueryOperation.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.opertion; + +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryJumpUrlOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.QueryJumpUrlResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.OnlyDevelopmentRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; + +public class WorkflowRefQueryOperation + extends AbstractDevelopmentOperation + implements RefQueryJumpUrlOperation { + + @Override + public QueryJumpUrlResponseRef query(OnlyDevelopmentRequestRef.QueryJumpUrlRequestRefImpl ref) { + // Now only support to fetch workflow open url, not support other orchestrations. + return ref.getDSSLabels().stream().filter(EnvDSSLabel.class::isInstance).findFirst().map(label -> { + String urlStr = "router/workflow/editable?labels=" + ((EnvDSSLabel) label).getEnv(); + return QueryJumpUrlResponseRef.newBuilder().setJumpUrl(urlStr).success(); + }).orElseThrow(() -> new ExternalOperationWarnException(50321, "Not exists EnvDSSLabel, cannot fetch orchestration open url.")); + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefUpdateOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefUpdateOperation.java new file mode 100644 index 000000000..b461d9497 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefUpdateOperation.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.opertion; + +import com.webank.wedatasphere.dss.common.protocol.JobStatus; +import com.webank.wedatasphere.dss.common.protocol.RequestUpdateWorkflow; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.ThirdlyRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseUpdateWorkflow; +import org.apache.linkis.rpc.Sender; + + +public class WorkflowRefUpdateOperation + extends AbstractDevelopmentOperation + implements RefUpdateOperation { + + private final Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(); + + @Override + public ResponseRef updateRef(ThirdlyRequestRef.UpdateRequestRefImpl requestRef) throws ExternalOperationFailedException { + String userName = requestRef.getUserName(); + long flowId = (long) requestRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY); + String flowName = (String) requestRef.getDSSJobContent().get(OrchestratorRefConstant.ORCHESTRATION_NAME); + String description = (String) requestRef.getDSSJobContent().get(OrchestratorRefConstant.ORCHESTRATION_DESCRIPTION); + String uses = (String) requestRef.getDSSJobContent().get(OrchestratorRefConstant.ORCHESTRATION_USES); + RequestUpdateWorkflow requestUpdateWorkflow = new RequestUpdateWorkflow(userName, flowId, flowName, description, uses); + ResponseUpdateWorkflow responseUpdateWorkflow = (ResponseUpdateWorkflow) sender.ask(requestUpdateWorkflow); + if(responseUpdateWorkflow.getJobStatus() == JobStatus.Success) { + return ResponseRef.newInternalBuilder().success(); + } else { + return ResponseRef.newInternalBuilder().error("Unknown error, please ask admin for help."); + } + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowCRUDService.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowCRUDService.java new file mode 100644 index 000000000..c6a0b655b --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowCRUDService.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.service; + +import com.webank.wedatasphere.dss.appconn.workflow.opertion.WorkflowRefCopyOperation; +import com.webank.wedatasphere.dss.appconn.workflow.opertion.WorkflowRefCreationOperation; +import com.webank.wedatasphere.dss.appconn.workflow.opertion.WorkflowRefDeletionOperation; +import com.webank.wedatasphere.dss.appconn.workflow.opertion.WorkflowRefUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefCRUDService; + + +public class WorkflowCRUDService extends AbstractRefCRUDService { + + @Override + protected WorkflowRefCreationOperation createRefCreationOperation() { + return new WorkflowRefCreationOperation(); + } + + @Override + protected WorkflowRefCopyOperation createRefCopyOperation() { + return new WorkflowRefCopyOperation(); + } + + @Override + protected WorkflowRefUpdateOperation createRefUpdateOperation() { + return new WorkflowRefUpdateOperation(); + } + + @Override + protected WorkflowRefDeletionOperation createRefDeletionOperation() { + return new WorkflowRefDeletionOperation(); + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowExportService.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowExportService.java new file mode 100644 index 000000000..7914f234d --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowExportService.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.service; + +import com.webank.wedatasphere.dss.appconn.workflow.opertion.WorkflowRefExportOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExportService; + +public class WorkflowExportService extends AbstractRefExportService { + + @Override + protected WorkflowRefExportOperation createRefExportOperation() { + return new WorkflowRefExportOperation(); + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowImportService.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowImportService.java new file mode 100644 index 000000000..8eb3238d1 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowImportService.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.service; + +import com.webank.wedatasphere.dss.appconn.workflow.opertion.WorkflowRefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefImportService; + +public class WorkflowImportService extends AbstractRefImportService { + + + @Override + @SuppressWarnings("unchecked") + protected WorkflowRefImportOperation createRefImportOperation() { + return new WorkflowRefImportOperation(); + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowQueryService.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowQueryService.java new file mode 100644 index 000000000..048991013 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/service/WorkflowQueryService.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.workflow.service; + +import com.webank.wedatasphere.dss.appconn.workflow.opertion.WorkflowRefQueryOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefQueryService; + +public class WorkflowQueryService extends AbstractRefQueryService { + + @Override + protected WorkflowRefQueryOperation createRefQueryOperation() { + return new WorkflowRefQueryOperation(); + } + +} diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/appconn.properties b/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/appconn.properties new file mode 100644 index 000000000..19365e2b5 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/appconn.properties @@ -0,0 +1,20 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + + + + + diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/log4j.properties b/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/log4j.properties new file mode 100644 index 000000000..ee8619595 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/log4j2.xml b/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8c40a73e8 --- /dev/null +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/resources/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-appconn/dss-appconn-core/pom.xml b/dss-appconn/dss-appconn-core/pom.xml new file mode 100644 index 000000000..72ff33882 --- /dev/null +++ b/dss-appconn/dss-appconn-core/pom.xml @@ -0,0 +1,106 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-appconn-core + + + 2.11.0 + 3.2.2 + + + + + com.webank.wedatasphere.dss + dss-standard-common + ${dss.version} + + + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-structure-integration-standard + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + + commons-collections + commons-collections + ${commons-collections.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-annotations + ${fasterxml.jackson.version} + provided + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-origin-sso-integration-standard + ${dss.version} + + + + + + \ No newline at end of file diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/AppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/AppConn.java new file mode 100644 index 000000000..add899c08 --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/AppConn.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core; + +import com.webank.wedatasphere.dss.appconn.core.exception.AppConnErrorException; +import com.webank.wedatasphere.dss.standard.common.core.AppStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; + +import java.util.List; + +public interface AppConn { + + void init() throws AppConnErrorException; + + /** + * 1. Get the dssappconnbean table record + * 2. Do a traversal to get all appinstances under each appconn + * 3. Instantiate the real appconn interface + */ + List getAppStandards(); + + AppDesc getAppDesc(); + + void setAppDesc(AppDesc appDesc); + +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/exception/AppConnErrorException.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/exception/AppConnErrorException.java new file mode 100644 index 000000000..11aec027d --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/exception/AppConnErrorException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.exception; + + +import org.apache.linkis.common.exception.ErrorException; + +public class AppConnErrorException extends ErrorException { + + public AppConnErrorException(int errorCode, String errorDesc){ + super(errorCode, errorDesc); + } + public AppConnErrorException(int errorCode, String errorDesc, Throwable cause){ + super(errorCode, errorDesc); + initCause(cause); + } + +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/exception/AppConnWarnException.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/exception/AppConnWarnException.java new file mode 100644 index 000000000..1019f0a3f --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/exception/AppConnWarnException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.exception; + +import org.apache.linkis.common.exception.WarnException; + +public class AppConnWarnException extends WarnException { + + public AppConnWarnException(int errCode, String desc) { + super(errCode, desc); + } + + public AppConnWarnException(int errCode, String desc, Throwable cause) { + super(errCode, desc); + initCause(cause); + } +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlyDevelopmentAppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlyDevelopmentAppConn.java new file mode 100644 index 000000000..6fa570293 --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlyDevelopmentAppConn.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.ext; + + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; + +/** + * 如果您的第三方系统想作为 DSS 工作流的一个节点集成进来,则需实现该 AppConn。 + * */ +public interface OnlyDevelopmentAppConn extends AppConn { + + DevelopmentIntegrationStandard getOrCreateDevelopmentStandard(); + +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlySSOAppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlySSOAppConn.java new file mode 100644 index 000000000..b927751fd --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlySSOAppConn.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.ext; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard; + +/** + * 如果您的第三方系统想与 DSS 完成 SSO 免登录跳转,则需使用该 AppConn 提供的能力。 + * + * {@code OnlySSOAppConn} 提供了默认的抽象类 {@code AbstractOnlySSOAppConn},该抽象类已提供了一级规范的默认实现。 + * + * 请注意:一般情况下,相关的 {@code AppConn} 子类会主动继承该抽象类,因此您无需实现 {@code OnlySSOAppConn} 的任何方法。 + * + * {@code OnlySSOAppConn} 的核心,是需要第三方 AppConn 按照要求引入 DSS 的 SSO Jar 包,完成相关接口的代码实现和引入。 + * + *
+ * + * 考虑到一种特殊情况:如果您的第三方系统只想与 DSS 实现 SSO 免登录跳转,那您完全无需重新写一个 OnlySSOAppConn 实现, + * 只需直接使用 {@code SSOAppConn}即可。您只需在 DSS 的 dss_appconn 表中新增一条记录,将 reference 字段指定为 + * sso 即可。更多关于 dss_appconn 表的介绍,请参考:第三方系统接入 DSS 开发指南#331-dss_appconn-表。 + */ +public interface OnlySSOAppConn extends AppConn { + + SSOIntegrationStandard getOrCreateSSOStandard(); + +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlyStructureAppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlyStructureAppConn.java new file mode 100644 index 000000000..74e6aec56 --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OnlyStructureAppConn.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.ext; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.standard.app.structure.StructureIntegrationStandard; + +/** + * 如果您的第三方系统想与 DSS 统一组织结构,如:工程的统一创建、更新、删除等管理操作,角色权限管理的统一管理,则需继承该 AppConn。 + */ +public interface OnlyStructureAppConn extends AppConn { + + StructureIntegrationStandard getOrCreateStructureStandard(); + +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OptionalAppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OptionalAppConn.java new file mode 100644 index 000000000..40bc0062c --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/OptionalAppConn.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.appconn.core.ext; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.standard.app.structure.OptionalIntegrationStandard; + +/** + * 这是一个可选的、非强制实现的 AppConn 规范,用于协助第三方 AppConn 提供一些特殊的 Operation 能力, + * 这些 Operation 与 DSS 的框架逻辑无关,并不是 DSS 框架要求第三方 AppConn 需具备的能力。 + * 这些可选的、非强制的 Operation 能力,通常是提供给 第三方 AppConn 内部的三大规范使用,或是 + * DSS 内置的一些应用工具,在使用这些第三方 AppConn 时,希望第三方 AppConn 提供的能力。 + *
+ * 所以,区别于三大规范,该 AppConn 规范的 Operation 不会在类头强制要求 RequestRef 和 ResponseRef 的类型, + * 内置的应用工具,可以在使用过程中,基于 OptionalOperation 这个顶层基类按需进行次级抽象定义,以及要求 + * 第三方 AppConn 按要求进行继承和实现。 + * @author enjoyyin + * @date 2022-03-18 + * @since 1.1.0 + */ +public interface OptionalAppConn extends AppConn { + + default OptionalIntegrationStandard getOrCreateOptionalStandard() { + return OptionalIntegrationStandard.getInstance(getAppDesc().getAppName()); + } + +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/SecondlyAppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/SecondlyAppConn.java new file mode 100644 index 000000000..bd932c0d5 --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/SecondlyAppConn.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.ext; + +/** + * {@code OnlySSOAppConn} 和 {@code OnlyStructureAppConn} 的子类。 + * 表示第三方系统既想打通 SSO,又想打通组织结构。 + */ +public interface SecondlyAppConn extends OnlySSOAppConn, OnlyStructureAppConn { + +} \ No newline at end of file diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/ThirdlyAppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/ThirdlyAppConn.java new file mode 100644 index 000000000..b5ae307b5 --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/ext/ThirdlyAppConn.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.ext; + +/** + * {@code OnlySSOAppConn}、{@code OnlyStructureAppConn} 和 {@code OnlyDevelopmentAppConn} 的子类。 + * 表示第三方系统既想打通 SSO 和组织结构,又想作为 DSS 工作流的一个节点集成进来。 + * */ +public interface ThirdlyAppConn extends OnlySSOAppConn, OnlyStructureAppConn, OnlyDevelopmentAppConn{ +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/impl/AbstractAppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/impl/AbstractAppConn.java new file mode 100644 index 000000000..e0a30c4b7 --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/impl/AbstractAppConn.java @@ -0,0 +1,107 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.impl; + + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.exception.AppConnErrorException; +import com.webank.wedatasphere.dss.appconn.core.exception.AppConnWarnException; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlySSOAppConn; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.core.AppIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.core.AppStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + + +public abstract class AbstractAppConn implements AppConn { + + private AppDesc appDesc; + private List appStandards; + protected final List appStandardMethodHeader = Arrays.asList("create", "getOrCreate", "get"); + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + public List getAppStandards() { + if(appStandards == null) { + synchronized (appStandardMethodHeader) { + if(appStandards == null) { + try { + init(); + } catch (AppConnErrorException e) { + throw new AppConnWarnException(e.getErrCode(), e.getMessage(), e); + } + } + } + } + return appStandards; + } + + protected abstract void initialize(); + + /** + * Specification: each appconn needs to define a method starting with create + * and returning as standard type to initialize the specifications owned by appconn. + * */ + @Override + public void init() throws AppConnErrorException { + initialize(); + appStandards = Arrays.stream(getClass().getDeclaredMethods()).map(method -> { + String methodName = method.getName(); + if(appStandardMethodHeader.stream().anyMatch(methodName::startsWith) && + AppStandard.class.isAssignableFrom(method.getReturnType())) { + try { + return (AppStandard) method.invoke(this); + } catch (ReflectiveOperationException e) { + logger.warn(methodName + " execute failed, ignore to set it into appStandardList of " + getClass().getSimpleName(), e); + } + } + return null; + }).filter(Objects::nonNull).collect(Collectors.toList()); + if(this instanceof OnlySSOAppConn) { + SSORequestService ssoRequestService = ((OnlySSOAppConn) this).getOrCreateSSOStandard().getSSORequestService(); + for(AppStandard appStandard : appStandards) { + if(appStandard instanceof AppIntegrationStandard) { + ((AppIntegrationStandard) appStandard).setSSORequestService(ssoRequestService); + } + try { + appStandard.init(); + } catch (AppStandardErrorException e) { + throw new AppConnErrorException(e.getErrCode(), "Init " + appStandard.getStandardName() + " failed!", e); + } + } + } + } + + @Override + public AppDesc getAppDesc() { + return appDesc; + } + + @Override + public void setAppDesc(AppDesc appDesc) { + this.appDesc = appDesc; + } + +} diff --git a/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/impl/AbstractOnlySSOAppConn.java b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/impl/AbstractOnlySSOAppConn.java new file mode 100644 index 000000000..f9a86d306 --- /dev/null +++ b/dss-appconn/dss-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/core/impl/AbstractOnlySSOAppConn.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.core.impl; + +import com.webank.wedatasphere.dss.appconn.core.exception.AppConnErrorException; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlySSOAppConn; +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandardFactory; +import com.webank.wedatasphere.dss.standard.app.sso.origin.OriginSSOIntegrationStandardFactory; +import com.webank.wedatasphere.dss.standard.app.sso.user.SSOUserService; +import com.webank.wedatasphere.dss.standard.app.sso.user.impl.SSOUserServiceImpl; +import com.webank.wedatasphere.dss.standard.common.utils.AppStandardClassUtils; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.LoggerFactory; + +public abstract class AbstractOnlySSOAppConn extends AbstractAppConn implements OnlySSOAppConn { + + private SSOIntegrationStandard SSO_INTEGRATION_STANDARD; + + @Override + public final void init() throws AppConnErrorException { + SSOIntegrationStandardFactory ssoIntegrationStandardFactory = + AppStandardClassUtils.getInstance(getAppDesc().getAppName()).getInstanceOrDefault(SSOIntegrationStandardFactory.class, new OriginSSOIntegrationStandardFactory()); + ssoIntegrationStandardFactory.init(); + SSO_INTEGRATION_STANDARD = ssoIntegrationStandardFactory.getSSOIntegrationStandard(); + LoggerFactory.getLogger(AbstractOnlySSOAppConn.class).info("For the first SSO Standard of {} AppConn, {} has created {}.", getAppDesc().getAppName(), + ssoIntegrationStandardFactory.getClass().getName(), SSO_INTEGRATION_STANDARD.getClass().getName()); + if(CollectionUtils.isNotEmpty(getAppDesc().getAppInstances())) { + getAppDesc().getAppInstances().forEach(appInstance -> { + SSOUserService ssoUserService = SSO_INTEGRATION_STANDARD.getSSOUserService(appInstance); + if(ssoUserService instanceof SSOUserServiceImpl) { + ((SSOUserServiceImpl) ssoUserService).setAppConnName(getAppDesc().getAppName()); + } + }); + } + // considering the plugin design model in different classloader, We must set it when each AppConn is instanced. + SSOHelper.setSSOBuilderService(SSO_INTEGRATION_STANDARD.getSSOBuilderService()); + super.init(); + } + + @Override + public final SSOIntegrationStandard getOrCreateSSOStandard() { + return SSO_INTEGRATION_STANDARD; + } +} diff --git a/dss-appconn/dss-appconn-loader/pom.xml b/dss-appconn/dss-appconn-loader/pom.xml new file mode 100644 index 000000000..968740a92 --- /dev/null +++ b/dss-appconn/dss-appconn-loader/pom.xml @@ -0,0 +1,86 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-appconn-loader + + + + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + **/*.yml + **/*.properties + **/*.sh + **/log4j2.xml + + + + + + + src/main/resources + + + + + \ No newline at end of file diff --git a/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/clazzloader/AppConnClassLoader.java b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/clazzloader/AppConnClassLoader.java new file mode 100644 index 000000000..7f2c60882 --- /dev/null +++ b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/clazzloader/AppConnClassLoader.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.loader.clazzloader; + +import java.net.URL; +import java.net.URLClassLoader; + +/** + * use UrlClassLoader load Jar package to jvm. + * */ +public class AppConnClassLoader extends URLClassLoader { + + public AppConnClassLoader(URL[] urls, ClassLoader parent) { + super(urls, parent); + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + return loadClass(name, false); + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + return super.loadClass(name, resolve); + } +} \ No newline at end of file diff --git a/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/AppConnLoader.java b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/AppConnLoader.java new file mode 100644 index 000000000..95df46e5c --- /dev/null +++ b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/AppConnLoader.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.loader.loader; + + +import com.webank.wedatasphere.dss.appconn.core.AppConn; + +/** + * Load interface specification of appconn + * */ +public interface AppConnLoader { + + AppConn getAppConn(String appConnName, String spi, String homePath) throws Exception; + +} \ No newline at end of file diff --git a/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/AppConnLoaderFactory.java b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/AppConnLoaderFactory.java new file mode 100644 index 000000000..c84f592c3 --- /dev/null +++ b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/AppConnLoaderFactory.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.loader.loader; + + +import com.webank.wedatasphere.dss.appconn.loader.conf.AppConnLoaderConf; +import org.apache.commons.lang.ClassUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AppConnLoaderFactory { + + private static final Logger logger = LoggerFactory.getLogger(AppConnLoaderFactory.class); + + private static Class clazz = CommonAppConnLoader.class; + private static AppConnLoader appConnLoader = null; + + @SuppressWarnings("unchecked") + public static AppConnLoader getAppConnLoader(){ + if (appConnLoader == null){ + synchronized (AppConnLoaderFactory.class){ + if (appConnLoader == null){ + // The corresponding classes can be loaded by configuration + String className = AppConnLoaderConf.CLASS_LOADER_CLASS_NAME().getValue(); + if (StringUtils.isNotBlank(className)){ + try{ + clazz = ClassUtils.getClass(className); + }catch(ClassNotFoundException e){ + logger.warn(String.format("Can not get AppConnLoader class %s, CommonAppConnLoader will be used by default.", className), e); + } + } + try { + appConnLoader = clazz.newInstance(); + } catch (Exception e) { + logger.error(String.format("Can not initialize AppConnLoader class %s.", clazz.getSimpleName()), e); + } + logger.info("Use {} to load all AppConns.", clazz.getSimpleName()); + } + } + } + return appConnLoader; + } + +} diff --git a/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/CommonAppConnLoader.java b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/CommonAppConnLoader.java new file mode 100644 index 000000000..ca04a6c24 --- /dev/null +++ b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/loader/CommonAppConnLoader.java @@ -0,0 +1,102 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.loader.loader; + + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.loader.clazzloader.AppConnClassLoader; +import com.webank.wedatasphere.dss.appconn.loader.exception.NoSuchAppConnException; +import com.webank.wedatasphere.dss.appconn.loader.utils.AppConnUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.standard.common.utils.AppStandardClassUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.BDPConfiguration; +import org.apache.linkis.common.exception.ErrorException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URL; +import java.nio.file.Paths; +import java.util.List; +import java.util.Properties; + +public class CommonAppConnLoader implements AppConnLoader { + + private static final String LIB_NAME = "lib"; + private static final String APP_CONN_PROPERTIES_NAME = "appconn.properties"; + + private static final Logger LOGGER = LoggerFactory.getLogger(CommonAppConnLoader.class); + + @Override + public AppConn getAppConn(String appConnName, String spi, String homePath) throws Exception { + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); + String libPathUrl; + if (StringUtils.isNotEmpty(homePath)){ + libPathUrl = new File(homePath, LIB_NAME).getPath(); + } else { + libPathUrl = Paths.get(AppConnUtils.getAppConnHomePath(), appConnName, LIB_NAME).toFile().getPath(); + } + LOGGER.info("The libPath url of AppConn {} is {}.", appConnName, libPathUrl); + List jars = AppConnUtils.getJarsUrlsOfPath(libPathUrl); + ClassLoader classLoader = AppStandardClassUtils.getClassLoader(appConnName, () -> new AppConnClassLoader(jars.toArray(new URL[1]), currentClassLoader)); + Thread.currentThread().setContextClassLoader(classLoader); + String fullClassName; + if (StringUtils.isEmpty(spi)) { + try { + fullClassName = AppConnUtils.getAppConnClassName(appConnName, libPathUrl, classLoader); + } catch (NoSuchAppConnException e) { + Thread.currentThread().setContextClassLoader(currentClassLoader); + throw e; + } + } else { + fullClassName = spi; + } + Class clazz = null; + try { + clazz = classLoader.loadClass(fullClassName); + } catch (ClassNotFoundException e) { + Thread.currentThread().setContextClassLoader(currentClassLoader); + DSSExceptionUtils.dealErrorException(70062, fullClassName + " class not found ", e, ErrorException.class); + } + AppConn retAppConn = (AppConn) clazz.newInstance(); + // 加载 appconn.properties + File appConnPropertiesPathUrl; + if (StringUtils.isNotEmpty(homePath)){ + appConnPropertiesPathUrl = new File(homePath, APP_CONN_PROPERTIES_NAME); + } else { + appConnPropertiesPathUrl = Paths.get(AppConnUtils.getAppConnHomePath(), appConnName, APP_CONN_PROPERTIES_NAME).toFile(); + } + if(appConnPropertiesPathUrl.exists() && appConnPropertiesPathUrl.isFile()) { + Properties properties = new Properties(); + properties.load(FileUtils.openInputStream(appConnPropertiesPathUrl)); + if(!properties.isEmpty()) { + LOGGER.info("AppConn {} try to load {}, the properties is {}.", appConnName, APP_CONN_PROPERTIES_NAME, properties); + properties.forEach((key, value) -> { + if(key != null && StringUtils.isNotBlank((String) key) && + value != null && StringUtils.isNotBlank((String) value)) { + BDPConfiguration.setIfNotExists((String) key, (String) value); + } + }); + } + } + Thread.currentThread().setContextClassLoader(currentClassLoader); + LOGGER.info("Loaded appConn {} with class {}.", appConnName, retAppConn.getClass().getName()); + return retAppConn; + } +} diff --git a/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/utils/AppConnUtils.java b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/utils/AppConnUtils.java new file mode 100644 index 000000000..bfab51f1f --- /dev/null +++ b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/utils/AppConnUtils.java @@ -0,0 +1,137 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.loader.utils; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.loader.exception.NoSuchAppConnException; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import org.apache.linkis.common.conf.CommonVars; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Modifier; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import org.apache.commons.lang.StringUtils; + +public class AppConnUtils { + + public static final String JAR_SUF_NAME = ".jar"; + + public static final String APPCONN_DIR_NAME = "dss-appconns"; + + public static final CommonVars APPCONN_HOME_PATH = CommonVars.apply("wds.dss.appconn.home.path", + new File(DSSCommonUtils.DSS_HOME.getValue(), APPCONN_DIR_NAME).getPath()); + + public static String getAppConnHomePath() { + return APPCONN_HOME_PATH.acquireNew(); + } + + /** + * Obtain the fully qualified name of the appconn to be instantiated. + * */ + public static String getAppConnClassName(String appConnName, String libPath, + ClassLoader classLoader) throws NoSuchAppConnException, IOException { + //1.Get all the jar packages under the directory + List jars = getJarsOfPath(libPath); + //2.Get the subclass of appconn from all jars + for (String jar : jars) { + for (String clazzName : getClassNameFrom(jar)) { + //3.Then find the subclass of appconn in the corresponding jar package + if (isChildClass(clazzName, AppConn.class, classLoader)) { + return clazzName; + } + } + } + throw new NoSuchAppConnException("Cannot find a appConn instance for AppConn " + appConnName + " in lib path " + libPath); + } + + public static List getJarsOfPath(String path) { + File file = new File(path); + List jars = new ArrayList<>(); + if (file.listFiles() != null) { + for (File f : file.listFiles()) { + // only search from dss-xxxxx.jar. + if (!f.isDirectory() && f.getName().endsWith(JAR_SUF_NAME) && f.getName().startsWith("dss-")) { + jars.add(f.getPath()); + } + } + } + return jars; + } + + + public static List getJarsUrlsOfPath(String path) throws MalformedURLException { + File file = new File(path); + List jars = new ArrayList<>(); + if (file.listFiles() != null) { + for (File f : file.listFiles()) { + if (!f.isDirectory() && f.getName().endsWith(JAR_SUF_NAME)) { + jars.add(f.toURI().toURL()); + } + } + } + return jars; + } + + + /** + * Then look for the subclass of appconn in the corresponding jar package, + * and read all the class file names from the jar package. + */ + private static List getClassNameFrom(String jarName) throws IOException { + List fileList = new ArrayList<>(); + JarFile jarFile = new JarFile(new File(jarName)); + Enumeration en = jarFile.entries(); + while (en.hasMoreElements()) { + String name1 = en.nextElement().getName(); + if (!name1.endsWith(".class")) { + continue; + } + String name2 = name1.substring(0, name1.lastIndexOf(".class")); + String name3 = name2.replaceAll("/", "."); + fileList.add(name3); + } + return fileList; + } + + + private static boolean isChildClass(String className, Class parentClazz, ClassLoader classLoader) { + if (StringUtils.isEmpty(className)) { + return false; + } + Class clazz = null; + try { + clazz = classLoader.loadClass(className); + //忽略抽象类和接口 + if (Modifier.isAbstract(clazz.getModifiers())) { + return false; + } + if (Modifier.isInterface(clazz.getModifiers())) { + return false; + } + } catch (Throwable t) { + return false; + } + return parentClazz.isAssignableFrom(clazz); + } + +} diff --git a/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/utils/ExceptionHelper.java b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/utils/ExceptionHelper.java new file mode 100644 index 000000000..7a8d97f45 --- /dev/null +++ b/dss-appconn/dss-appconn-loader/src/main/java/com/webank/wedatasphere/dss/appconn/loader/utils/ExceptionHelper.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.loader.utils; + +import org.apache.linkis.common.exception.ErrorException; + +public class ExceptionHelper { + public static void dealErrorException(int errorCode, String errorMsg, Throwable t) throws ErrorException { + ErrorException errorException = new ErrorException(errorCode, errorMsg); + errorException.initCause(t); + throw errorException; + } +} diff --git a/dss-appconn/dss-appconn-loader/src/main/scala/com/webank/wedatasphere/dss/appconn/loader/conf/AppConnLoaderConf.scala b/dss-appconn/dss-appconn-loader/src/main/scala/com/webank/wedatasphere/dss/appconn/loader/conf/AppConnLoaderConf.scala new file mode 100644 index 000000000..da9e59a26 --- /dev/null +++ b/dss-appconn/dss-appconn-loader/src/main/scala/com/webank/wedatasphere/dss/appconn/loader/conf/AppConnLoaderConf.scala @@ -0,0 +1,23 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.loader.conf + +import org.apache.linkis.common.conf.CommonVars + +object AppConnLoaderConf { + val CLASS_LOADER_CLASS_NAME = CommonVars("dss.appconn.loader.classname", "") +} diff --git a/dss-appconn/dss-appconn-loader/src/main/scala/com/webank/wedatasphere/dss/appconn/loader/exception/NoSuchAppConnException.scala b/dss-appconn/dss-appconn-loader/src/main/scala/com/webank/wedatasphere/dss/appconn/loader/exception/NoSuchAppConnException.scala new file mode 100644 index 000000000..00a36f599 --- /dev/null +++ b/dss-appconn/dss-appconn-loader/src/main/scala/com/webank/wedatasphere/dss/appconn/loader/exception/NoSuchAppConnException.scala @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.loader.exception + +import org.apache.linkis.common.exception.ErrorException + +case class NoSuchAppConnException(errDesc:String) extends ErrorException(70059, errDesc) + diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml new file mode 100644 index 000000000..26df350a4 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml @@ -0,0 +1,88 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-appconn-manager-client + + + + com.webank.wedatasphere.dss + dss-appconn-manager-core + ${dss.version} + + + org.apache.linkis + linkis-bml-client + ${linkis.version} + + + commons-beanutils + commons-beanutils + + + linkis-common + org.apache.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + **/*.yml + **/*.properties + **/*.sh + **/log4j2.xml + + + + + + + src/main/resources + + + + + \ No newline at end of file diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/conf/AppConnManagerClientConfiguration.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/conf/AppConnManagerClientConfiguration.java new file mode 100644 index 000000000..c95e3ed53 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/conf/AppConnManagerClientConfiguration.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class AppConnManagerClientConfiguration { + + public final static CommonVars DSS_APPCONN_CLIENT_TOKEN = CommonVars.apply("wds.dss.appconn.client.user.token","WS-AUTH"); + public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.dss.appconn.client.user","ws"); + + + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppConnInfoImpl.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppConnInfoImpl.java new file mode 100644 index 000000000..6de52d81f --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppConnInfoImpl.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.entity; + +import com.webank.wedatasphere.dss.common.entity.Resource; + +public class AppConnInfoImpl implements AppConnInfo { + + private String appConnName; + private String className; + private Resource appConnResource; + private String reference; + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + public void setClassName(String className) { + this.className = className; + } + + public void setAppConnResource(Resource appConnResource) { + this.appConnResource = appConnResource; + } + + @Override + public String getAppConnName() { + return appConnName; + } + + @Override + public String getClassName() { + return className; + } + + @Override + public Resource getAppConnResource() { + return appConnResource; + } + + @Override + public String getReference() { + return reference; + } + + public void setReference(String reference) { + this.reference = reference; + } + + @Override + public String toString() { + return "AppConnInfoImpl{" + + "appConnName='" + appConnName + '\'' + + ", className='" + className + '\'' + + ", appConnResource=" + appConnResource + + ", reference='" + reference + '\'' + + '}'; + } +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppInstanceInfoImpl.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppInstanceInfoImpl.java new file mode 100644 index 000000000..56c678657 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppInstanceInfoImpl.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.entity; + +public class AppInstanceInfoImpl implements AppInstanceInfo { + + private Long id; + private String url; + private String homepageUri; + private String labels; + private String enhanceJson; + + @Override + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Override + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + @Override + public String getHomepageUri() { + return homepageUri; + } + + public void setHomepageUri(String homepageUri) { + this.homepageUri = homepageUri; + } + + @Override + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + @Override + public String getEnhanceJson() { + return enhanceJson; + } + + public void setEnhanceJson(String enhanceJson) { + this.enhanceJson = enhanceJson; + } + + @Override + public String toString() { + return "AppInstanceInfoImpl{" + + "id=" + id + + ", url='" + url + '\'' + + ", homepageUrl='" + homepageUri + '\'' + + ", labels='" + labels + '\'' + + ", enhanceJson='" + enhanceJson + '\'' + + '}'; + } +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/exception/AppConnHomeNotExistsWarnException.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/exception/AppConnHomeNotExistsWarnException.java new file mode 100644 index 000000000..441aca022 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/exception/AppConnHomeNotExistsWarnException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; + + +public class AppConnHomeNotExistsWarnException extends DSSRuntimeException { + + public AppConnHomeNotExistsWarnException(int errCode, String desc) { + super(errCode, desc); + } + + public AppConnHomeNotExistsWarnException(int errCode, String desc, Throwable t) { + super(errCode, desc); + initCause(t); + } +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnInfoServiceImpl.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnInfoServiceImpl.java new file mode 100644 index 000000000..8927d2eb6 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnInfoServiceImpl.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.service; + +import com.webank.wedatasphere.dss.appconn.manager.action.GetAppConnInfoAction; +import com.webank.wedatasphere.dss.appconn.manager.action.GetAppConnInfosAction; +import com.webank.wedatasphere.dss.appconn.manager.action.GetAppInstancesAction; +import com.webank.wedatasphere.dss.appconn.manager.conf.AppConnManagerClientConfiguration; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppInstanceInfo; +import com.webank.wedatasphere.dss.appconn.manager.result.GetAppConnInfoResult; +import com.webank.wedatasphere.dss.appconn.manager.result.GetAppConnInfosResult; +import com.webank.wedatasphere.dss.appconn.manager.result.GetAppInstancesResult; +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.httpclient.config.ClientConfig; +import org.apache.linkis.httpclient.dws.DWSHttpClient; +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import java.util.List; + +import static com.webank.wedatasphere.dss.appconn.manager.conf.AppConnManagerClientConfiguration.LINKIS_ADMIN_USER; + +public class AppConnInfoServiceImpl implements AppConnInfoService { + + private ClientConfig clientConfig = DWSClientConfigBuilder.newBuilder().setDWSVersion("v1").addServerUrl(Configuration.getGateWayURL()) + .connectionTimeout(300000).discoveryEnabled(false).setAuthenticationStrategy(new TokenAuthenticationStrategy()).setAuthTokenKey(LINKIS_ADMIN_USER.getValue()) + .setAuthTokenValue(AppConnManagerClientConfiguration.DSS_APPCONN_CLIENT_TOKEN.getValue()).maxConnectionSize(50).readTimeout(300000).build(); + private DWSHttpClient client = new DWSHttpClient((DWSClientConfig) clientConfig, "AppConn-Client-"); + + @Override + public List getAppConnInfos() { + GetAppConnInfosAction getAppConnInfosAction =new GetAppConnInfosAction(); + getAppConnInfosAction.setUser(LINKIS_ADMIN_USER.getValue()); + GetAppConnInfosResult result = (GetAppConnInfosResult) client.execute(getAppConnInfosAction); + return result.getAppConnInfos(); + } + + @Override + public AppConnInfo getAppConnInfo(String appConnName) { + GetAppConnInfoAction action = new GetAppConnInfoAction(); + action.setAppConnName(appConnName); + action.setUser(LINKIS_ADMIN_USER.getValue()); + GetAppConnInfoResult result = (GetAppConnInfoResult) client.execute(action); + return result.getAppConnInfo(); + } + + @Override + public List getAppInstancesByAppConnInfo(AppConnInfo appConnInfo) { + return getAppInstancesByAppConnName(appConnInfo.getAppConnName()); + } + + @Override + public List getAppInstancesByAppConnName(String appConnName) { + GetAppInstancesAction action = new GetAppInstancesAction(); + action.setAppConnName(appConnName); + action.setUser(LINKIS_ADMIN_USER.getValue()); + GetAppInstancesResult result = (GetAppInstancesResult) client.execute(action); + return result.getAppInstanceInfos(); + } +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java new file mode 100644 index 000000000..a29a31347 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java @@ -0,0 +1,125 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.service; + +import com.webank.wedatasphere.dss.appconn.loader.utils.AppConnUtils; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import com.webank.wedatasphere.dss.appconn.manager.exception.AppConnHomeNotExistsWarnException; +import com.webank.wedatasphere.dss.appconn.manager.utils.AppConnIndexFileUtils; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class AppConnResourceServiceImpl implements AppConnResourceService { + + private static final Logger LOGGER = LoggerFactory.getLogger(AppConnResourceServiceImpl.class); + + private BmlClient bmlClient = BmlClientFactory.createBmlClient(); + + @Override + public String getAppConnHome(AppConnInfo appConnInfo) { + File appConnHomePath = new File(AppConnUtils.getAppConnHomePath()); + String appConnName = appConnInfo.getAppConnName(); + if(!appConnHomePath.exists()) { + throw new AppConnHomeNotExistsWarnException(20350, "AppConn home path " + appConnHomePath.getPath() + " not exists."); + } else if (!appConnHomePath.isDirectory()) { + throw new AppConnHomeNotExistsWarnException(20350, "AppConn home path " + appConnHomePath.getPath() + " is not a directory."); + } + File appConnPath = new File(appConnHomePath, appConnName); + Resource resource = appConnInfo.getAppConnResource(); + if(!appConnPath.exists() && !appConnPath.mkdir()) { + throw new AppConnHomeNotExistsWarnException(20350, "Cannot create dir " + appConnPath.getPath() + " for AppConn " + appConnName); + } + if(AppConnIndexFileUtils.isLatestIndex(appConnPath, resource)) { + return appConnPath.getPath(); + } + if (StringUtils.isNotBlank(appConnInfo.getReference())) { + return appConnPath.getPath(); + } + LOGGER.info("Try to download latest resource {} in version {} from BML for AppConn {}.", resource.getResourceId(), + resource.getVersion(), appConnName); + // At first, Download AppConn files from bml. + String zipFilePath = new File(appConnHomePath, appConnName + "_" + resource.getVersion() + ".zip").getPath(); + bmlClient.downloadResource(Utils.getJvmUser(), resource.getResourceId(), resource.getVersion(), + "file://" + zipFilePath, true); + // Then, try to unzip it. + if(!deleteAppConnDir(appConnPath)) { + throw new AppConnHomeNotExistsWarnException(20350, "Cannot delete dir " + appConnPath.getPath() + " for AppConn " + appConnName); + } + try { + ZipHelper.unzip(zipFilePath); + } catch (DSSErrorException e) { + throw new AppConnHomeNotExistsWarnException(20350, "Unzip " + zipFilePath + " failed, AppConn " + appConnName, e); + } + + File oldIndexFile = AppConnIndexFileUtils.getIndexFile(appConnPath); + + // update index file. + if (oldIndexFile != null && !oldIndexFile.delete()) { + throw new AppConnHomeNotExistsWarnException(20350, "Delete index file " + oldIndexFile.getName() + " failed, please ensure the permission is all right."); + } + +// TODO ZipUtils.fileToUnzip(zipFilePath, appConnHomePath.getPath()); + // Only reserve latest 2 version files. + File[] historyZipFiles = appConnHomePath.listFiles((p, fileName) -> fileName.startsWith(appConnName) && fileName.endsWith(".zip")); + if(historyZipFiles.length > 2) { + List files = Arrays.stream(historyZipFiles).sorted().collect(Collectors.toList()); + // ignore delete failed. + IntStream.range(0, files.size() - 2).forEach(index -> files.get(index).delete()); + } + // Finally, write index file. + Path indexFile = Paths.get(appConnPath.getPath(), AppConnIndexFileUtils.getIndexFileName(resource)); + try { + Files.createFile(indexFile); + } catch (IOException e) { + throw new AppConnHomeNotExistsWarnException(20350, "Cannot create index file " + indexFile.toFile().getPath() + " for AppConn " + + appConnName, e); + } + return appConnPath.getPath(); + } + + public boolean deleteAppConnDir(File f){ + if(f.isDirectory()){ + File[] files = f.listFiles(); + for (File key : files) { + if(key.isFile()){ + key.delete(); + }else{ + deleteAppConnDir(key); + } + } + } + return f.delete(); + } + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/action/GetAppConnInfosAction.scala b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/action/GetAppConnInfosAction.scala new file mode 100644 index 000000000..29b7e058c --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/action/GetAppConnInfosAction.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.action + + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.{GetAction, UserAction} + +class GetAppConnInfosAction extends GetAction with DWSHttpAction with UserAction{ + + private var user:String = _ + + override def suffixURLs: Array[String] = Array("dss", "framework", "project", "appconn", "listAppConnInfos") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user +} + +class GetAppConnInfoAction extends GetAction with DWSHttpAction with UserAction { + + private var appConnName: String = _ + private var user:String = _ + + override def suffixURLs: Array[String] = Array("dss", "framework", "project", "appconn", appConnName, "get") + + def setAppConnName(appConnName: String): Unit = this.appConnName = appConnName + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} \ No newline at end of file diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/action/GetAppInstancesAction.scala b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/action/GetAppInstancesAction.scala new file mode 100644 index 000000000..e1ee1ff1b --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/action/GetAppInstancesAction.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.action + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.{GetAction, UserAction} + +class GetAppInstancesAction extends GetAction with DWSHttpAction with UserAction { + + private var user:String = _ + + private var appConnName: String = _ + + override def suffixURLs: Array[String] = Array("dss", "framework", "project", "appconn", appConnName, "getAppInstances") + + def setAppConnName(appConnName: String): Unit = this.appConnName = appConnName + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppConnInfoResult.scala b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppConnInfoResult.scala new file mode 100644 index 000000000..9e1fb2d2f --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppConnInfoResult.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.result + +import java.util + +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfoImpl +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/framework/project/appconn/[a-zA-Z0-9]+/get") +class GetAppConnInfoResult extends DWSResult { + + private var appConnInfo: AppConnInfoImpl = _ + + def setAppConnInfo(appConnInfo: util.Map[String, Object]): Unit = { + val json = DSSCommonUtils.COMMON_GSON.toJson(appConnInfo) + this.appConnInfo = DSSCommonUtils.COMMON_GSON.fromJson(json, classOf[AppConnInfoImpl]) + } + + def getAppConnInfo: AppConnInfoImpl = appConnInfo + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppConnInfosResult.scala b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppConnInfosResult.scala new file mode 100644 index 000000000..87f0212da --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppConnInfosResult.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.result + +import java.util + +import com.webank.wedatasphere.dss.appconn.manager.entity.{AppConnInfo, AppConnInfoImpl} +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.collection.convert.wrapAsScala._ +import scala.collection.convert.wrapAsJava._ + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/framework/project/appconn/listAppConnInfos") +class GetAppConnInfosResult extends DWSResult { + + private var appConnInfos: util.List[AppConnInfo] = _ + + def setAppConnInfos(appConnInfos: util.List[util.Map[String, Object]]): Unit = { + this.appConnInfos = appConnInfos.map{ map => + val json = DSSCommonUtils.COMMON_GSON.toJson(map) + DSSCommonUtils.COMMON_GSON.fromJson(json, classOf[AppConnInfoImpl]).asInstanceOf[AppConnInfo] + } + } + + def getAppConnInfos: util.List[AppConnInfo] = appConnInfos + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppInstancesResult.scala b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppInstancesResult.scala new file mode 100644 index 000000000..12edbfbc2 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/scala/com/webank/wedatasphere/dss/appconn/manager/result/GetAppInstancesResult.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.result + +import java.util + +import com.webank.wedatasphere.dss.appconn.manager.entity.{AppInstanceInfo, AppInstanceInfoImpl} +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.collection.convert.wrapAsJava._ +import scala.collection.convert.wrapAsScala._ + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/framework/project/appconn/[a-zA-Z0-9\\-]+/getAppInstances") +class GetAppInstancesResult extends DWSResult { + + private var appInstanceInfos: util.List[AppInstanceInfo] = _ + + def setAppInstanceInfos(appInstanceInfos: util.List[util.Map[String, Object]]): Unit = { + this.appInstanceInfos = appInstanceInfos.map{ map => + val json = DSSCommonUtils.COMMON_GSON.toJson(map) + DSSCommonUtils.COMMON_GSON.fromJson(json, classOf[AppInstanceInfoImpl]) + } + } + + def getAppInstanceInfos: util.List[AppInstanceInfo] = appInstanceInfos + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml new file mode 100644 index 000000000..dfc2bbbe5 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml @@ -0,0 +1,73 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-appconn-manager-core + jar + + + + com.webank.wedatasphere.dss + dss-appconn-loader + ${dss.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + -target:jvm-1.8 + + + + org.apache.maven.plugins + maven-jar-plugin + + + **/*.yml + **/*.properties + **/*.sh + **/log4j2.xml + + + + + + + src/main/resources + + + + + \ No newline at end of file diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/AppConnManager.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/AppConnManager.java new file mode 100644 index 000000000..d99f3665e --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/AppConnManager.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.exception.AppConnWarnException; +import com.webank.wedatasphere.dss.appconn.manager.impl.AbstractAppConnManager; + +import java.util.List; +import java.util.stream.Collectors; + + +public interface AppConnManager { + + /** + * init AppConnManager + */ + void init(); + + /** + * get all AppConns + */ + List listAppConns(); + + default List listAppConns(Class appConnClass) { + return listAppConns().stream().filter(appConnClass::isInstance).map(appConn -> (T) appConn).collect(Collectors.toList()); + } + + /** + * Returns an appconn by its name + */ + AppConn getAppConn(String appConnName); + + default T getAppConn(Class appConnClass) { + List appConns = listAppConns().stream().filter(appConnClass::isInstance).collect(Collectors.toList()); + if(appConns.isEmpty()) { + throw new AppConnWarnException(25344, "Cannot find a AppConn instance for " + appConnClass.getSimpleName()); + } else if(appConns.size() > 1) { + throw new AppConnWarnException(25344, "More than one AppConn instances is found, list: " + appConns); + } + return (T) appConns.get(0); + } + + void reloadAppConn(String appConnName); + + String getAppConnHomePath(String appConnName); + + static void setLazyLoad() { + AbstractAppConnManager.setLazyLoad(); + } + + static AppConnManager getAppConnManager() { + return AbstractAppConnManager.getAppConnManager(); + } + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppConnInfo.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppConnInfo.java new file mode 100644 index 000000000..9308dbb88 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppConnInfo.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.entity; + +import com.webank.wedatasphere.dss.common.entity.Resource; + +public interface AppConnInfo { + + String getAppConnName(); + + String getClassName(); + + Resource getAppConnResource(); + + String getReference(); + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppInstanceInfo.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppInstanceInfo.java new file mode 100644 index 000000000..9e0a6bac2 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/entity/AppInstanceInfo.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.entity; + +public interface AppInstanceInfo { + + Long getId(); + + String getUrl(); + + String getHomepageUri(); + + String getLabels(); + + String getEnhanceJson(); + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/exception/AppConnIndexFileWarnException.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/exception/AppConnIndexFileWarnException.java new file mode 100644 index 000000000..75884f42e --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/exception/AppConnIndexFileWarnException.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; + +public class AppConnIndexFileWarnException extends DSSRuntimeException { + + public AppConnIndexFileWarnException(int errCode, String desc) { + super(errCode, desc); + } +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java new file mode 100644 index 000000000..9340464c0 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java @@ -0,0 +1,278 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.loader.loader.AppConnLoader; +import com.webank.wedatasphere.dss.appconn.loader.loader.AppConnLoaderFactory; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppInstanceInfo; +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnInfoService; +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnResourceService; +import com.webank.wedatasphere.dss.appconn.manager.utils.AppInstanceConstants; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.standard.common.desc.AppDescImpl; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstanceImpl; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +public abstract class AbstractAppConnManager implements AppConnManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractAppConnManager.class); + private final AppConnLoader appConnLoader = AppConnLoaderFactory.getAppConnLoader(); + + private final Map appConns = new HashMap<>(); + private boolean isLoaded = false; + private List appConnList = null; + AppConnInfoService appConnInfoService; + private AppConnResourceService appConnResourceService; + private AppConnRefreshThread appConnRefreshThread; + + private static AppConnManager appConnManager; + private static boolean lazyLoad = false; + + public static void setLazyLoad() { + lazyLoad = true; + } + + public static AppConnManager getAppConnManager() { + if (appConnManager != null) { + return appConnManager; + } + synchronized (AbstractAppConnManager.class) { + if (appConnManager == null) { + appConnManager = ClassUtils.getInstanceOrDefault(AppConnManager.class, new AppConnManagerImpl()); + LOGGER.info("The instance of AppConnManager is {}.", appConnManager.getClass().getName()); + appConnManager.init(); + } + } + return appConnManager; + } + + @Override + public void init() { + appConnInfoService = createAppConnInfoService(); + LOGGER.info("The instance of AppConnInfoService is {}.", appConnInfoService.getClass().getName()); + appConnResourceService = createAppConnResourceService(); + LOGGER.info("The instance of AppConnResourceService is {}.", appConnResourceService.getClass().getName()); + if (!lazyLoad && !isLoaded) { + loadAppConns(); + isLoaded = true; + } + } + + protected abstract AppConnInfoService createAppConnInfoService(); + + protected abstract AppConnResourceService createAppConnResourceService(); + + protected void loadAppConns() { + LOGGER.info("Begin to init all AppConns."); + List appConnInfos = appConnInfoService.getAppConnInfos(); + if (appConnInfos == null || appConnInfos.isEmpty()) { + if (appConns.isEmpty()) { + throw new DSSRuntimeException("No AppConnInfos returned when the first time init AppConnList."); + } + LOGGER.warn("No AppConnInfos returned, ignore it."); + return; + } + long refreshInterval = AppInstanceConstants.APP_CONN_REFRESH_INTERVAL.getValue().toLong(); + appConnRefreshThread = new AppConnRefreshThread(this, appConnInfos); + Utils.defaultScheduler().scheduleAtFixedRate(appConnRefreshThread, refreshInterval, refreshInterval, TimeUnit.MILLISECONDS); + Map appConns = new HashMap<>(); + Consumer loadAndAdd = DSSExceptionUtils.handling(appConnInfo -> { + AppConn appConn = loadAppConn(appConnInfo); + if (appConn != null) { + appConns.put(appConnInfo.getAppConnName().toLowerCase(), appConn); + } + }); + appConnInfos.forEach(DSSExceptionUtils.handling(appConnInfo -> { + if (appConns.containsKey(appConnInfo.getAppConnName())) { + return; + } + if (StringUtils.isNotBlank(appConnInfo.getReference())) { + if (!appConns.containsKey(appConnInfo.getReference())) { + AppConnInfo referenceAppConnInfo = appConnInfos.stream().filter(info -> info.getAppConnName().equals(appConnInfo.getReference())) + .findAny().orElseThrow(() -> new DSSRuntimeException("cannot find the reference appConn " + appConnInfo.getReference() + + " for appConn " + appConnInfo.getAppConnName())); + loadAndAdd.accept(referenceAppConnInfo); + } + AppConn appConn = loadAppConn(appConnInfo, appConns.get(appConnInfo.getReference())); + appConns.put(appConnInfo.getAppConnName().toLowerCase(), appConn); + } else { + loadAndAdd.accept(appConnInfo); + } + })); + synchronized (this.appConns) { + this.appConns.clear(); + this.appConns.putAll(appConns); + appConnList = Collections.unmodifiableList(new ArrayList<>(appConns.values())); + } + LOGGER.info("Inited all AppConns, the AppConn list are {}.", this.appConns.keySet()); + } + + protected AppConn loadAppConn(AppConnInfo appConnInfo) throws Exception { + LOGGER.info("Ready to load AppConn {}, the appConnInfo are {}.", appConnInfo.getAppConnName(), appConnInfo); + String appConnHome = appConnResourceService.getAppConnHome(appConnInfo); + LOGGER.info("Try to load AppConn {} with home path {}.", appConnInfo.getAppConnName(), appConnHome); + AppConn appConn = appConnLoader.getAppConn(appConnInfo.getAppConnName(), + appConnInfo.getClassName(), appConnHome); + AppDescImpl appDesc = loadAppDesc(appConnInfo); + appConn.setAppDesc(appDesc); + appConn.init(); + LOGGER.info("AppConn {} is loaded successfully.", appConnInfo.getAppConnName()); + return appConn; + } + + protected AppConn loadAppConn(AppConnInfo appConnInfo, AppConn referenceAppConn) throws Exception { + LOGGER.info("Ready to load AppConn {} with referenceAppConn {}, the appConnInfo are {}.", appConnInfo.getAppConnName(), + referenceAppConn.getClass().getSimpleName(), appConnInfo); + if (appConnInfo.getAppConnResource() != null) { + String appConnHome = appConnResourceService.getAppConnHome(appConnInfo); + LOGGER.warn("Because AppConn {} is a referenced AppConn, we only download its resources(since not null) to home path {}, but never load AppConn by it.", + appConnInfo.getAppConnName(), appConnHome); + } + AppDescImpl appDesc = loadAppDesc(appConnInfo); + AppConn appConn = referenceAppConn.getClass().newInstance(); + appConn.setAppDesc(appDesc); + appConn.init(); + LOGGER.info("AppConn {} is loaded successfully.", appConnInfo.getAppConnName()); + return appConn; + } + + protected AppDescImpl loadAppDesc(AppConnInfo appConnInfo) { + List instanceInfos = appConnInfoService.getAppInstancesByAppConnInfo(appConnInfo); + LOGGER.info("The instanceInfos of AppConn {} are {}.", appConnInfo.getAppConnName(), instanceInfos); + AppDescImpl appDesc = new AppDescImpl(); + for (AppInstanceInfo instanceBean : instanceInfos) { + AppInstanceImpl appInstance = new AppInstanceImpl(); + copyProperties(appConnInfo.getAppConnName(), instanceBean, appInstance); + appDesc.addAppInstance(appInstance); + } + if (appDesc.getAppInstances().isEmpty()) { + LOGGER.warn("The AppConn {} has no appInstance, maybe this AppConn is a reference AppConn? If not, please check the database info.", appConnInfo.getAppConnName()); + } + appDesc.setAppName(appConnInfo.getAppConnName()); + return appDesc; + } + + private void copyProperties(String appConnName, AppInstanceInfo appInstanceInfo, AppInstanceImpl appInstance) { + appInstance.setBaseUrl(appInstanceInfo.getUrl()); + Map config = new HashMap<>(); + if (StringUtils.isNotEmpty(appInstanceInfo.getEnhanceJson())) { + try { + config = DSSCommonUtils.COMMON_GSON.fromJson(appInstanceInfo.getEnhanceJson(), Map.class); + } catch (Exception e) { + LOGGER.error("The json of AppConn {} is not a correct json. content: {}.", appConnName, appInstanceInfo.getEnhanceJson(), e); + throw new DSSRuntimeException("The json of AppConn " + appConnName + " is not a correct json."); + } + } + appInstance.setConfig(config); + if (StringUtils.isNotBlank(appInstanceInfo.getHomepageUri())) { + appInstance.setHomepageUri(appInstanceInfo.getHomepageUri()); + } + //TODO should use Linkis labelFactory to new labels. + List labels = Arrays.stream(appInstanceInfo.getLabels().split(",")).map(EnvDSSLabel::new) + .collect(Collectors.toList()); + appInstance.setLabels(labels); + appInstance.setId(appInstanceInfo.getId()); + } + + private void lazyLoadAppConns() { + if (lazyLoad && !isLoaded) { + synchronized (this.appConns) { + if (lazyLoad && !isLoaded) { + loadAppConns(); + } + } + } + } + + @Override + public List listAppConns() { + lazyLoadAppConns(); + return appConnList; + } + + + @Override + public AppConn getAppConn(String appConnName) { + lazyLoadAppConns(); + return appConns.get(appConnName.toLowerCase()); + } + + @Override + public void reloadAppConn(String appConnName) { + AppConnInfo appConnInfo = appConnInfoService.getAppConnInfo(appConnName); + if (appConnInfo == null) { + throw new DSSRuntimeException("Cannot get any info about AppConn " + appConnName); + } + reloadAppConn(appConnInfo); + } + + @Override + public String getAppConnHomePath(String appConnName) { + AppConnInfo appConnInfo = appConnRefreshThread.getAppConnInfos().stream().filter(info -> info.getAppConnName().equals(appConnName)) + .findAny().orElseThrow(() -> new DSSRuntimeException("Not exists AppConn " + appConnName)); + return appConnResourceService.getAppConnHome(appConnInfo); + } + + public void reloadAppConn(AppConnInfo appConnInfo) { + lazyLoadAppConns(); + AppConn appConn; + try { + if (StringUtils.isNotBlank(appConnInfo.getReference())) { + AppConn referenceAppConn = getAppConn(appConnInfo.getReference()); + if (referenceAppConn == null) { + throw new DSSRuntimeException("Load AppConn " + appConnInfo.getAppConnName() + + " failed! Caused by: The reference AppConn " + appConnInfo.getReference() + " is not exists."); + } + appConn = loadAppConn(appConnInfo, referenceAppConn); + } else { + appConn = loadAppConn(appConnInfo); + } + } catch (DSSRuntimeException e) { + throw e; + } catch (Exception e) { + LOGGER.error("Reload AppConn failed.", e); + DSSRuntimeException exception = new DSSRuntimeException("Load AppConn " + appConnInfo.getAppConnName() + " failed!"); + exception.initCause(e); + throw exception; + } + if (appConn == null) { + return; + } + synchronized (this.appConns) { + this.appConns.put(appConnInfo.getAppConnName().toLowerCase(), appConn); + appConnList = Collections.unmodifiableList(new ArrayList<>(appConns.values())); + } + LOGGER.info("Reloaded AppConn {}.", appConnInfo.getAppConnName()); + } + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AppConnManagerImpl.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AppConnManagerImpl.java new file mode 100644 index 000000000..c06025d8e --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AppConnManagerImpl.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.impl; + +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnInfoService; +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnResourceService; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.utils.ClassUtils; + + +public class AppConnManagerImpl extends AbstractAppConnManager { + + @Override + protected AppConnInfoService createAppConnInfoService() { + try { + return ClassUtils.getInstance(AppConnInfoService.class); + } catch (DSSErrorException e) { + throw new DSSRuntimeException(25000, "Cannot find a useful AppConnInfoService.", e); + } + } + + @Override + protected AppConnResourceService createAppConnResourceService() { + try { + return ClassUtils.getInstance(AppConnResourceService.class); + } catch (DSSErrorException e) { + throw new DSSRuntimeException(25000, "Cannot find a useful AppConnResourceService.", e); + } + } + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AppConnRefreshThread.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AppConnRefreshThread.java new file mode 100644 index 000000000..406bda7e8 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AppConnRefreshThread.java @@ -0,0 +1,74 @@ +package com.webank.wedatasphere.dss.appconn.manager.impl; + +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Optional; + +/** + * @author enjoyyin + * @date 2022-04-08 + * @since 1.1.0 + */ +public class AppConnRefreshThread implements Runnable { + + private static final Logger LOGGER = LoggerFactory.getLogger(AppConnRefreshThread.class); + + private AbstractAppConnManager appConnManager; + private volatile List appConnInfos; + + public AppConnRefreshThread(AbstractAppConnManager appConnManager, + List appConnInfos) { + this.appConnManager = appConnManager; + this.appConnInfos = appConnInfos; + } + + public List getAppConnInfos() { + return appConnInfos; + } + + @Override + public void run() { + LOGGER.info("try to refresh all AppConns."); + List appConnInfos; + try { + appConnInfos = appConnManager.appConnInfoService.getAppConnInfos(); + } catch (Exception e) { + LOGGER.warn("Fetch appConn infos failed, ignore to refresh all AppConns.", e); + return; + } + if(appConnInfos == null || appConnInfos.isEmpty()) { + LOGGER.warn("no appConnInfos fetched, ignore to refresh it."); + return; + } + LOGGER.info("Fetched appConn infos list => {}.", appConnInfos); + appConnInfos.forEach(appConnInfo -> { + Optional oldOne = this.appConnInfos.stream().filter(old -> old.getAppConnName().equals(appConnInfo.getAppConnName())).findAny(); + if(!oldOne.isPresent() || isChanged(oldOne.get(), appConnInfo)) { + LOGGER.warn("AppConn info {} has updated, now try to refresh it.", appConnInfo.getAppConnName()); + try { + appConnManager.reloadAppConn(appConnInfo); + } catch (Exception e) { + // If update failed, it seems like some error happened in this AppConn. + // this AppConn will not be refreshed any more, unless the admin changes the AppConn plugin files to optimize it. + LOGGER.warn("Reload AppConn {} failed, ignore it", appConnInfo.getAppConnName(), e); + } + } + }); + // now, do not support to delete exists AppConn, since deletion operation is very dangerous. + this.appConnInfos = appConnInfos; + LOGGER.info("all AppConns have refreshed."); + } + + private boolean isChanged(AppConnInfo one, AppConnInfo other) { + if(one.getAppConnResource() == null && other.getAppConnResource() == null) { + return false; + } else if(one.getAppConnResource() == null || other.getAppConnResource() == null) { + return true; + } else { + return !one.getAppConnResource().equals(other.getAppConnResource()); + } + } +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnInfoService.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnInfoService.java new file mode 100644 index 000000000..56c85f3b4 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnInfoService.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.service; + +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppInstanceInfo; + +import java.io.Serializable; +import java.util.List; + + +public interface AppConnInfoService{ + + List getAppConnInfos(); + + AppConnInfo getAppConnInfo(String appConnName); + + List getAppInstancesByAppConnInfo(AppConnInfo appConnInfo); + + List getAppInstancesByAppConnName(String appConnName); + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceService.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceService.java new file mode 100644 index 000000000..f6615f57b --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceService.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.service; + +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; + +import java.io.Serializable; + +public interface AppConnResourceService{ + + String getAppConnHome(AppConnInfo appConnInfo); + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppConnIndexFileUtils.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppConnIndexFileUtils.java new file mode 100644 index 000000000..01c0278be --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppConnIndexFileUtils.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.utils; + +import com.webank.wedatasphere.dss.appconn.manager.exception.AppConnIndexFileWarnException; +import com.webank.wedatasphere.dss.common.entity.Resource; +import java.io.File; +import java.util.Arrays; + +import static com.webank.wedatasphere.dss.appconn.manager.utils.AppInstanceConstants.*; + +public class AppConnIndexFileUtils { + + private static boolean isIndexFile(String fileName) { + return fileName.startsWith(INDEX_FILE_PREFIX) && fileName.endsWith(INDEX_FILE_SUFFIX); + } + + public static File getIndexFile(File parent) { + File[] indexFiles = parent.listFiles((p, fileName) -> isIndexFile(fileName)); + if(indexFiles == null) { + return null; + }else if(indexFiles.length > 1) { + throw new AppConnIndexFileWarnException(20533, "More than one index files exists, indexFile list: " + Arrays.asList(indexFiles)); + } else if(indexFiles.length == 0) { + return null; + } else { + return indexFiles[0]; + } + } + + public static boolean isLatestIndex(File parent, Resource resource) { + File indexFile = getIndexFile(parent); + if(indexFile == null) { + return false; + } + String version = indexFile.getName().replaceAll(INDEX_FILE_PREFIX, "").replaceAll(INDEX_FILE_SUFFIX, ""); + return resource.getVersion().equals(version); + } + + public static String getIndexFileName(Resource resource) { + return INDEX_FILE_PREFIX + resource.getVersion() + INDEX_FILE_SUFFIX; + } + +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppConnManagerUtils.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppConnManagerUtils.java new file mode 100644 index 000000000..1d1a65f79 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppConnManagerUtils.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.dss.appconn.manager.utils; + +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +/** + * @author enjoyyin + * @date 2022-06-24 + * @since 1.1.0 + */ +public class AppConnManagerUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(AppConnManagerUtils.class); + + /** + * 主动加载所有的 AppConn,用于提高性能 + */ + public static void autoLoadAppConnManager() { + TimeType delay = CommonVars.apply("wds.dss.appconn.auto-load.delay", new TimeType("20m")).getValue(); + LOGGER.info("We will try to auto-load all AppConns if no usage lasting {}.", delay.toString()); + Utils.defaultScheduler().schedule(() -> { + LOGGER.info("Try to auto-load all AppConns since no usage lasting {}.", delay.toString()); + try { + AppConnManager.getAppConnManager(); + } catch (Exception e) { + LOGGER.info("Load AppConns failed.", e); + return; + } + LOGGER.info("All AppConns have loaded successfully."); + }, delay.toLong(), TimeUnit.MILLISECONDS); + } +} diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppInstanceConstants.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppInstanceConstants.java new file mode 100644 index 000000000..398c53878 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/utils/AppInstanceConstants.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.manager.utils; + +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; + +public class AppInstanceConstants { + + static final String INDEX_FILE_PREFIX = "index_"; + static final String INDEX_FILE_SUFFIX = ".index"; + + static final String REQUEST_URI = "reqUri"; + + public static final CommonVars APP_CONN_REFRESH_INTERVAL = CommonVars.apply("wds.dss.appconn.refresh.interval", new TimeType("5m")); + + public static String getHomepageUrl(AppInstance appInstance, SSOUrlBuilderOperation ssoUrlBuilderOperation, + Long workspaceId, String workspaceName) { + String homepageUrl = getHomepageUrl(appInstance.getBaseUrl(), appInstance.getHomepageUri(), + workspaceId, workspaceName); + if(MapUtils.isEmpty(appInstance.getConfig()) || ssoUrlBuilderOperation == null || + !appInstance.getConfig().containsKey(REQUEST_URI)) { + return homepageUrl; + } else { + String reqUri = (String) appInstance.getConfig().get(REQUEST_URI); + String reqUrl; + if(appInstance.getBaseUrl().endsWith("/")) { + reqUrl = appInstance.getBaseUrl() + reqUri; + } else { + reqUrl = appInstance.getBaseUrl() + "/" + reqUri; + } + ssoUrlBuilderOperation.redirectTo(homepageUrl); + ssoUrlBuilderOperation.setReqUrl(reqUrl); + return ssoUrlBuilderOperation.getBuiltUrl(); + } + } + + public static String getHomepageUrl(String baseUrl, String homepageUri, + Long workspaceId, String workspaceName) { + if(StringUtils.isBlank(homepageUri)) { + return baseUrl; + } + if(workspaceId != null && homepageUri.contains("${workspaceId}")) { + homepageUri = homepageUri.replace("${workspaceId}", workspaceId.toString()); + } + if(StringUtils.isNotBlank(workspaceName) && homepageUri.contains("${workspaceName}")) { + homepageUri = homepageUri.replace("${workspaceName}", workspaceName); + } + if(baseUrl.endsWith("/")) { + return baseUrl + homepageUri; + } else { + return baseUrl + "/" + homepageUri; + } + } + +} diff --git a/dss-appconn/dss-appconn-manager/pom.xml b/dss-appconn/dss-appconn-manager/pom.xml new file mode 100644 index 000000000..ec4db13c9 --- /dev/null +++ b/dss-appconn/dss-appconn-manager/pom.xml @@ -0,0 +1,36 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-appconn-manager + pom + + + dss-appconn-manager-core + dss-appconn-manager-client + + + \ No newline at end of file diff --git a/dss-appconn/dss-scheduler-appconn/pom.xml b/dss-appconn/dss-scheduler-appconn/pom.xml new file mode 100644 index 000000000..531dc3ad9 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/pom.xml @@ -0,0 +1,74 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-scheduler-appconn + + + + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + + + jackson-databind + com.fasterxml.jackson.core + + + + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + + + com.webank.wedatasphere.dss + dss-structure-integration-standard + ${dss.version} + + + com.webank.wedatasphere.dss + dss-workflow-conversion-standard + ${dss.version} + compile + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + \ No newline at end of file diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/AbstractSchedulerAppConn.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/AbstractSchedulerAppConn.java new file mode 100644 index 000000000..92fba7170 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/AbstractSchedulerAppConn.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.scheduler; + +import com.webank.wedatasphere.dss.appconn.core.impl.AbstractOnlySSOAppConn; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.AbstractConversionIntegrationStandard; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ConversionIntegrationStandard; +import com.webank.wedatasphere.dss.workflow.conversion.ProjectConversionIntegrationStandard; + +public abstract class AbstractSchedulerAppConn extends AbstractOnlySSOAppConn implements SchedulerAppConn { + + private ConversionIntegrationStandard conversionIntegrationStandard; + + protected ConversionIntegrationStandard createConversionIntegrationStandard() { + return new ProjectConversionIntegrationStandard(); + } + + @Override + protected void initialize() { + conversionIntegrationStandard = createConversionIntegrationStandard(); + if(conversionIntegrationStandard instanceof AbstractConversionIntegrationStandard) { + ((AbstractConversionIntegrationStandard) conversionIntegrationStandard).setAppConn(this); + } + } + + @Override + public ConversionIntegrationStandard getOrCreateConversionStandard() { + return conversionIntegrationStandard; + } +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/AbstractSchedulerStructureIntegrationStandard.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/AbstractSchedulerStructureIntegrationStandard.java new file mode 100644 index 000000000..5cbcd0dd4 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/AbstractSchedulerStructureIntegrationStandard.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.appconn.scheduler; + +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationService; +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public abstract class AbstractSchedulerStructureIntegrationStandard + extends AbstractStructureIntegrationStandard + implements SchedulerStructureIntegrationStandard { + + /** + * 统一编排规范,用于打通 DSS 与 调度系统的编排体系。 + * 例如:打通 DSS 工作流 与 Schedulis 工作流。 + * 请注意,如果对接的 SchedulerAppConn 系统本身不支持管理工作流,则无需实现该接口。 + * @return 如果对应的调度系统不支持管理工作流,则返回 null;否则返回具体的实现类 + */ + protected OrchestrationService createOrchestrationService() { + return null; + } + + @Override + public final OrchestrationService getOrchestrationService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createOrchestrationService, OrchestrationService.class); + } +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/SchedulerAppConn.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/SchedulerAppConn.java new file mode 100644 index 000000000..5564d0a52 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/SchedulerAppConn.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.scheduler; + +import com.webank.wedatasphere.dss.appconn.core.ext.OnlySSOAppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyStructureAppConn; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ConversionIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.structure.StructureIntegrationStandard; +import com.webank.wedatasphere.dss.workflow.conversion.WorkflowConversionIntegrationStandard; + +public interface SchedulerAppConn extends OnlySSOAppConn, OnlyStructureAppConn { + + /** + * 默认将提供 ProjectConversionIntegrationStandard + * @return + */ + ConversionIntegrationStandard getOrCreateConversionStandard(); + + @Override + SchedulerStructureIntegrationStandard getOrCreateStructureStandard(); +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/SchedulerStructureIntegrationStandard.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/SchedulerStructureIntegrationStandard.java new file mode 100644 index 000000000..8c92a5c6f --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/SchedulerStructureIntegrationStandard.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.appconn.scheduler; + +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationService; +import com.webank.wedatasphere.dss.standard.app.structure.StructureIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public interface SchedulerStructureIntegrationStandard extends StructureIntegrationStandard { + + /** + * 统一编排规范,用于打通 DSS 与 调度系统的编排体系。 + * 例如:打通 DSS 工作流 与 Schedulis 工作流。 + * 请注意,如果对接的 SchedulerAppConn 系统本身不支持管理工作流,则无需实现该接口。 + * @return 如果对应的调度系统不支持管理工作流,则返回 null;否则返回具体的实现类 + */ + OrchestrationService getOrchestrationService(AppInstance appInstance); + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationCreationOperation.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationCreationOperation.java new file mode 100644 index 000000000..5d51b1244 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationCreationOperation.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration; + +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.DSSOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + + +public interface OrchestrationCreationOperation> + extends StructureOperation { + + /** + * Try to create the one-to-one related refOrchestration in third-party AppConn. + * If created successfully, please return a OrchestrationResponseRef contained refOrchestrationId, + * so DSS can use the refOrchestrationId to operate the related refOrchestration in third-party AppConn. + * The returned refOrchestrationId is the other OrchestrationOperations which used. + * @param orchestrationRef contains the DSS orchestration info. + * @return a OrchestrationResponseRef contained refOrchestrationId + * @throws ExternalOperationFailedException + */ + OrchestrationResponseRef createOrchestration(R orchestrationRef) throws ExternalOperationFailedException; + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationDeletionOperation.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationDeletionOperation.java new file mode 100644 index 000000000..b2540a24b --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationDeletionOperation.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration; + +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.RefOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public interface OrchestrationDeletionOperation> + extends StructureOperation { + + /** + * delete the related refOrchestration in third-party AppConn by refOrchestrationId + * which returned by OrchestrationCreationOperation. + * refOrchestrationId must not be null, please use it to delete the refOrchestration. + * @param requestRef refOrchestration info, refOrchestrationId must not be null + * @return the result of deletion, just success or failure. + * @throws ExternalOperationFailedException + */ + ResponseRef deleteOrchestration(R requestRef); + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationSearchOperation.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationSearchOperation.java new file mode 100644 index 000000000..e1f007555 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationSearchOperation.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration; + +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationResponseRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.RefOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public interface OrchestrationSearchOperation> + extends StructureOperation { + + /** + * Try to search the orchestration by provide refOrchestrationId or orchestrationName. + * If refOrchestrationId is not null, then the third-part AppConn should use refOrchestrationId to search the refOrchestration; + * otherwise, the third-part AppConn should use orchestrationName to search the refOrchestration. + *
+ * If the refOrchestration is exists, please return a OrchestrationResponseRef which has been set in the refOrchestrationId; + * otherwise, just return an empty succeeded OrchestrationResponseRef. + * @param requestRef the refOrchestration info + * @return return a OrchestrationResponseRef contained refOrchestrationId if the refOrchestration is exists, otherwise + * just return an empty succeeded OrchestrationResponseRef. + * @throws ExternalOperationFailedException If some error are happened. + */ + OrchestrationResponseRef searchOrchestration(R requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationService.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationService.java new file mode 100644 index 000000000..b179c94d6 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationService.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration; + +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureService; + +/** + * 统一编排规范,专门用于打通 DSS 与 SchedulerAppConn(调度系统)的编排体系。 + * 例如:打通 DSS 工作流 与 Schedulis 工作流。 + * 请注意,如果对接的 SchedulerAppConn 系统本身不支持管理工作流,则无需实现该接口。 + * @author enjoyyin + * @date 2022-03-14 + * @since 1.1.0 + */ +public abstract class OrchestrationService extends AbstractStructureService { + + public final OrchestrationCreationOperation getOrchestrationCreationOperation() { + return getOrCreate(this::createOrchestrationCreationOperation, OrchestrationCreationOperation.class); + } + + protected abstract OrchestrationCreationOperation createOrchestrationCreationOperation(); + + public final OrchestrationUpdateOperation getOrchestrationUpdateOperation() { + return getOrCreate(this::createOrchestrationUpdateOperation, OrchestrationUpdateOperation.class); + } + + protected abstract OrchestrationUpdateOperation createOrchestrationUpdateOperation(); + + public final OrchestrationDeletionOperation getOrchestrationDeletionOperation() { + return getOrCreate(this::createOrchestrationDeletionOperation, OrchestrationDeletionOperation.class); + } + + protected abstract OrchestrationDeletionOperation createOrchestrationDeletionOperation(); + + + public final OrchestrationSearchOperation getOrchestrationSearchOperation() { + return getOrCreate(this::createOrchestrationSearchOperation, OrchestrationSearchOperation.class); + } + + protected abstract OrchestrationSearchOperation createOrchestrationSearchOperation(); + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationUpdateOperation.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationUpdateOperation.java new file mode 100644 index 000000000..5268fbf90 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/OrchestrationUpdateOperation.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration; + +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + + +public interface OrchestrationUpdateOperation> + extends StructureOperation { + + /** + * Try to update the related refOrchestration in third-party AppConn. + * Usually, DSS only want to update the orchestrationName, description and permissions info in third-party refOrchestration, + * so the refOrchestrationId is always exists and can not be changeable. + *
+ * Notice: do not try to change the refOrchestrationId already related with the third-party refOrchestration. + * @param orchestrationRef contains the DSS Orchestration info updated. + * @return the result of update, just success or failure. + * @throws ExternalOperationFailedException + */ + ResponseRef updateOrchestration(R orchestrationRef) throws ExternalOperationFailedException; + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/DSSOrchestrationContentRequestRef.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/DSSOrchestrationContentRequestRef.java new file mode 100644 index 000000000..a937e830e --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/DSSOrchestrationContentRequestRef.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestration; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public interface DSSOrchestrationContentRequestRef> + extends RefProjectContentRequestRef, StructureRequestRef { + + default DSSOrchestration getDSSOrchestration() { + return (DSSOrchestration) getParameter("dssOrchestration"); + } + + default R setDSSOrchestration(DSSOrchestration dssOrchestration) { + setParameter("dssOrchestration", dssOrchestration); + return (R) this; + } + + class DSSOrchestrationContentRequestRefImpl extends StructureRequestRefImpl + implements DSSOrchestrationContentRequestRef {} + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/OrchestrationResponseRef.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/OrchestrationResponseRef.java new file mode 100644 index 000000000..761101f9e --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/OrchestrationResponseRef.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public interface OrchestrationResponseRef extends ResponseRef { + + /** + * 对应于SchedulerAppConn(调度系统)的编排模式(一般就是调度系统的工作流)。 + * @return 返回调度系统的编排ID + */ + Long getRefOrchestrationId(); + + static ExternalBuilder newExternalBuilder() { + return new ExternalBuilder(); + } + + class ExternalBuilder extends ResponseRefBuilder.ExternalResponseRefBuilder { + private Long refOrchestrationId; + public ExternalBuilder setRefOrchestrationId(Long refOrchestrationId) { + this.refOrchestrationId = refOrchestrationId; + return this; + } + @Override + protected OrchestrationResponseRef createResponseRef() { + return new OrchestrationResponseRefImpl(); + } + + class OrchestrationResponseRefImpl extends ResponseRefImpl implements OrchestrationResponseRef { + public OrchestrationResponseRefImpl() { + super(ExternalBuilder.this.responseBody, ExternalBuilder.this.status, + ExternalBuilder.this.errorMsg, ExternalBuilder.this.responseMap); + } + @Override + public Long getRefOrchestrationId() { + return refOrchestrationId; + } + } + } + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/OrchestrationUpdateRequestRef.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/OrchestrationUpdateRequestRef.java new file mode 100644 index 000000000..a70103c04 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/OrchestrationUpdateRequestRef.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public interface OrchestrationUpdateRequestRef> + extends DSSOrchestrationContentRequestRef, RefOrchestrationContentRequestRef { + + @Override + default String getOrchestrationName() { + return getDSSOrchestration().getName(); + } + + @Override + default R setOrchestrationName(String orchestrationName) { + if(getDSSOrchestration() instanceof DSSOrchestratorInfo) { + ((DSSOrchestratorInfo) getDSSOrchestration()).setName(orchestrationName); + } + return (R) this; + } + + class OrchestrationUpdateRequestRefImpl extends StructureRequestRefImpl + implements OrchestrationUpdateRequestRef {} + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/RefOrchestrationContentRequestRef.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/RefOrchestrationContentRequestRef.java new file mode 100644 index 000000000..8ee2bdc8e --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/structure/orchestration/ref/RefOrchestrationContentRequestRef.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public interface RefOrchestrationContentRequestRef> + extends RefProjectContentRequestRef, StructureRequestRef { + + default Long getRefOrchestrationId() { + return (Long) getParameter("refOrchestrationId"); + } + + default R setRefOrchestrationId(Long refOrchestrationId) { + setParameter("refOrchestrationId", refOrchestrationId); + return (R) this; + } + + default String getOrchestrationName() { + return (String) getParameter("orchestrationName"); + } + + default R setOrchestrationName(String orchestrationName) { + setParameter("orchestrationName", orchestrationName); + return (R) this; + } + + class RefOrchestrationContentRequestRefImpl extends StructureRequestRefImpl + implements RefOrchestrationContentRequestRef {} + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/utils/OrchestrationOperationUtils.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/utils/OrchestrationOperationUtils.java new file mode 100644 index 000000000..76640b4a1 --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/utils/OrchestrationOperationUtils.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.appconn.scheduler.utils; + +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationService; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.DSSOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.RefOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; + +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +import static com.webank.wedatasphere.dss.standard.app.structure.utils.StructureOperationUtils.tryStructureOperation; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 1.1.0 + */ +public class OrchestrationOperationUtils { + + public static V tryOrchestrationOperation(Supplier getOrchestrationService, + Function getOrchestrationOperation, + Consumer dssOrchestrationContentRequestRefConsumer, + Consumer refOrchestrationContentRequestRefConsumer, + BiFunction responseRefConsumer, + String errorMsg) { + return tryStructureOperation(getOrchestrationService, getOrchestrationOperation, structureRequestRef -> { + if(dssOrchestrationContentRequestRefConsumer != null && structureRequestRef instanceof DSSOrchestrationContentRequestRef) { + dssOrchestrationContentRequestRefConsumer.accept((DSSOrchestrationContentRequestRef) structureRequestRef); + } + if(refOrchestrationContentRequestRefConsumer != null && structureRequestRef instanceof RefOrchestrationContentRequestRef) { + refOrchestrationContentRequestRefConsumer.accept((RefOrchestrationContentRequestRef) structureRequestRef); + } + }, responseRefConsumer, errorMsg); + } + +} diff --git a/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/utils/SchedulerConf.java b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/utils/SchedulerConf.java new file mode 100644 index 000000000..f8163b6ba --- /dev/null +++ b/dss-appconn/dss-scheduler-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/scheduler/utils/SchedulerConf.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.appconn.scheduler.utils; + +import org.apache.linkis.common.conf.CommonVars; + +/** + * @author enjoyyin + * @date 2022-04-07 + * @since 1.1.0 + */ +public class SchedulerConf { + + public static final CommonVars JOB_LABEL = CommonVars.apply("wds.dss.appconn.scheduler.job.label", "dev"); + +} diff --git a/dss-appconn/linkis-appconn-engineplugin/pom.xml b/dss-appconn/linkis-appconn-engineplugin/pom.xml new file mode 100644 index 000000000..e7572b08b --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/pom.xml @@ -0,0 +1,187 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + linkis-appconn-engineplugin + + + + org.apache.linkis + linkis-engineconn-plugin-core + ${linkis.version} + + + org.apache.linkis + linkis-computation-engineconn + ${linkis.version} + + + reflections + org.reflections + + + jackson-core + com.fasterxml.jackson.core + + + javassist + org.javassist + + + + + + org.apache.linkis + linkis-rpc + ${linkis.version} + + + hibernate-validator + org.hibernate.validator + + + provided + + + + org.apache.linkis + linkis-storage + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-scheduler-appconn + ${dss.version} + + + spring-beans + org.springframework + + + + + + com.webank.wedatasphere.dss + dss-appconn-manager-client + ${dss.version} + + + + + com.webank.wedatasphere.dss + dss-development-process-standard-execution + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + + + + javax.mail + mail + 1.4 + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + -target:jvm-1.8 + + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + linkis-engineplugin-appconn + false + false + + src/main/assembly/distribution.xml + + + + + + + ${basedir}/src/main/resources + + **/*.properties + **/*.xml + **/*.yml + + + + ${project.artifactId}-${project.version} + + + \ No newline at end of file diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/assembly/distribution.xml b/dss-appconn/linkis-appconn-engineplugin/src/main/assembly/distribution.xml new file mode 100644 index 000000000..4ebadd9af --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/assembly/distribution.xml @@ -0,0 +1,81 @@ + + + + linkis-enginePlugin-appconn + + dir + zip + + true + linkis-engineplugin-appconn + + + + + + /dist/v1/lib + true + true + false + true + true + + + + + + + ${basedir}/src/main/resources + + enginePlugin.properties + + 0777 + / + unix + + + + ${basedir}/src/main/resources + + *.properties + log4j2*.xml + + 0777 + /dist/v1/conf + unix + + + + ${basedir}/target + + *.jar + + + *doc.jar + + /plugin/1 + + + + + + diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/java/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnExecutionUtils.java b/dss-appconn/linkis-appconn-engineplugin/src/main/java/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnExecutionUtils.java new file mode 100644 index 000000000..6f5543bdc --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/java/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnExecutionUtils.java @@ -0,0 +1,43 @@ +package org.apache.linkis.manager.engineplugin.appconn.executor; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.RefExecutionService; +import com.webank.wedatasphere.dss.standard.app.development.utils.DevelopmentOperationUtils; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; + +import java.util.List; +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-18 + * @since 0.5.0 + */ +public class AppConnExecutionUtils { + + public static ResponseRef tryToOperation(RefExecutionService refExecutionService, String contextId, String projectName, + ExecutionRequestRefContextImpl executionRequestRefContext, List dssLabels, + String name, String type, String userName, Workspace workspace, + Map refJobContent, Map variables) { + return DevelopmentOperationUtils.tryRefJobContentRequestRefOperation(() -> refExecutionService, + developmentService -> refExecutionService.getRefExecutionOperation(), + refJobContentRequestRef -> refJobContentRequestRef.setRefJobContent(refJobContent), + dssContextRequestRef -> dssContextRequestRef.setContextId(contextId), + projectRefRequestRef -> projectRefRequestRef.setProjectName(projectName), + (developmentOperation, developmentRequestRef) -> { + RefExecutionRequestRef requestRef = (RefExecutionRequestRef) developmentRequestRef; + requestRef.setExecutionRequestRefContext(executionRequestRefContext) + .setVariables(variables) + .setDSSLabels(dssLabels) + .setName(name) + .setType(type) + .setUserName(userName) + .setWorkspace(workspace); + return ((RefExecutionOperation) developmentOperation).execute(requestRef); + }, "execute node " + name + " with type " + type); + } + +} diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/resources/linkis-engineconn.properties b/dss-appconn/linkis-appconn-engineplugin/src/main/resources/linkis-engineconn.properties new file mode 100644 index 000000000..5e3cb3eab --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/resources/linkis-engineconn.properties @@ -0,0 +1,57 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +wds.linkis.server.version=v1 + +wds.linkis.engineconn.spark.conf.enable=true + +wds.linkis.engineconn.debug.enable=true + +#wds.linkis.keytab.enable=true + +wds.linkis.engineconn.plugin.default.class=org.apache.linkis.manager.engineplugin.appconn.AppConnEngineConnPlugin + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.workflow.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.framework.appconn.dao + + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/${databaseName}?characterEncoding=UTF-8 + +wds.linkis.server.mybatis.datasource.username= + +wds.linkis.server.mybatis.datasource.password= + +wds.linkis.gateway.ip=127.0.0.1 +wds.linkis.gateway.port=9001 + +wds.linkis.gateway.url=http://127.0.0.1:9001/ + +wds.linkis.mysql.is.encrypt=false + +wds.linkis.reflect.scan.package=org.apache.linkis,com.webank.wedatasphere.dss +spring.spring.mvc.servlet.path=/api/rest_j/v1 + +spring.spring.servlet.multipart.max-file-size=200MB +spring.spring.servlet.multipart.max-request-size=200MB + +wds.linkis.engineconn.support.parallelism=true + +wds.linkis.engineconn.max.free.time=0 \ No newline at end of file diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/resources/log4j2.xml b/dss-appconn/linkis-appconn-engineplugin/src/main/resources/log4j2.xml new file mode 100644 index 000000000..062ae6e20 --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/resources/log4j2.xml @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/AppConnEngineConnPlugin.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/AppConnEngineConnPlugin.scala new file mode 100644 index 000000000..fea56f4e1 --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/AppConnEngineConnPlugin.scala @@ -0,0 +1,77 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn + +import java.util + +import org.apache.linkis.manager.engineplugin.appconn.factory.AppConnEngineConnFactory +import org.apache.linkis.manager.engineplugin.appconn.launch.AppConnProcessEngineConnLaunchBuilder +import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin +import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory +import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder +import org.apache.linkis.manager.engineplugin.common.resource.{EngineResourceFactory, GenericEngineResourceFactory} +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel} +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator + +class AppConnEngineConnPlugin extends EngineConnPlugin { + + private val resourceLocker = new Object() + + private val engineLaunchBuilderLocker = new Object() + + private val engineFactoryLocker = new Object() + + private var engineResourceFactory: EngineResourceFactory = _ + + private var engineLaunchBuilder: EngineConnLaunchBuilder = _ + + private var engineFactory: EngineConnFactory = _ + + private val defaultLabels: util.List[Label[_]] = new util.ArrayList[Label[_]]() + + + override def init(params: util.Map[String, Any]): Unit = { + val engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(EngineType.APPCONN.toString) + this.defaultLabels.add(engineTypeLabel) + } + + override def getEngineResourceFactory: EngineResourceFactory = { + if (null == engineResourceFactory) resourceLocker.synchronized { + if (null == engineResourceFactory) { + engineResourceFactory = new GenericEngineResourceFactory + } + } + engineResourceFactory + } + + override def getEngineConnLaunchBuilder: EngineConnLaunchBuilder = { + new AppConnProcessEngineConnLaunchBuilder + } + + override def getEngineConnFactory: EngineConnFactory = { + if (null == engineFactory) engineFactoryLocker.synchronized { + if (null == engineFactory) { + engineFactory = new AppConnEngineConnFactory + } + } + engineFactory + } + + override def getDefaultLabels: util.List[Label[_]] = defaultLabels + +} diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/conf/AppConnEngineConnConfiguration.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/conf/AppConnEngineConnConfiguration.scala new file mode 100644 index 000000000..974ffb086 --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/conf/AppConnEngineConnConfiguration.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn.conf + +import org.apache.linkis.common.conf.CommonVars + + +object AppConnEngineConnConfiguration { + + val GATEWAY_SPRING_APPLICATION = CommonVars("wds.linkis.gateway.spring.name", "dataworkcloud-gateway") + + val CONCURRENT_LIMIT = CommonVars("wds.linkis.engineconn.appconn.conncurrent.limit", 100) + + val ENGINE_DEFAULT_LIMIT = CommonVars("wds.linkis.jdbc.default.limit", 5000) + +} diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/exception/AppConnExecuteFailedException.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/exception/AppConnExecuteFailedException.scala new file mode 100644 index 000000000..df8d2f5b1 --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/exception/AppConnExecuteFailedException.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn.exception + +import org.apache.linkis.common.exception.ErrorException + +case class AppConnExecuteFailedException(errCode: Int, desc: String) extends ErrorException(errCode, desc) diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AbstractExecutionRequestRefContext.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AbstractExecutionRequestRefContext.scala new file mode 100644 index 000000000..3d91c6019 --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AbstractExecutionRequestRefContext.scala @@ -0,0 +1,104 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn.executor + +import java.util + +import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext +import com.webank.wedatasphere.dss.standard.app.development.listener.exception.AppConnExecutionErrorException +import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader, ResultSetWriter} +import org.apache.linkis.common.io.{FsPath, MetaData, Record} +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext +import org.apache.linkis.manager.engineplugin.appconn.conf.AppConnEngineConnConfiguration +import org.apache.linkis.rpc.Sender +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader} + +abstract class AbstractExecutionRequestRefContext(engineExecutorContext: EngineExecutionContext, + user: String, + submitUser: String) + extends ExecutionRequestRefContext with Logging { + + override def getRuntimeMap: util.Map[String, AnyRef] = engineExecutorContext.getProperties + + override def appendLog(log: String): Unit = engineExecutorContext.appendStdout(log) + + override def updateProgress(progress: Float): Unit = engineExecutorContext.pushProgress(progress, Array.empty) + + override def getSubmitUser: String = submitUser + + override def getUser: String = user + + override def createTableResultSetWriter[M <: MetaData, R <: Record](): ResultSetWriter[M, R] = + createTableResultSetWriter(null) + + override def createTableResultSetWriter[M <: MetaData, R <: Record](resultSetAlias: String): ResultSetWriter[M, R] = + createResultSetWriter(ResultSetFactory.TABLE_TYPE, resultSetAlias) + + override def createTextResultSetWriter[M <: MetaData, R <: Record](): ResultSetWriter[M, R] = createTextResultSetWriter(null) + + override def createTextResultSetWriter[M <: MetaData, R <: Record](resultSetAlias: String): ResultSetWriter[M, R] = + createResultSetWriter(ResultSetFactory.TEXT_TYPE, resultSetAlias) + + override def createHTMLResultSetWriter[M <: MetaData, R <: Record](): ResultSetWriter[M, R] = createHTMLResultSetWriter(null) + + override def createHTMLResultSetWriter[M <: MetaData, R <: Record](resultSetAlias: String): ResultSetWriter[M, R] = + createResultSetWriter(ResultSetFactory.HTML_TYPE, resultSetAlias) + + override def createPictureResultSetWriter[M <: MetaData, R <: Record](): ResultSetWriter[M, R] = createPictureResultSetWriter(null) + + override def createPictureResultSetWriter[M <: MetaData, R <: Record](resultSetAlias: String): ResultSetWriter[M, R] = + createResultSetWriter(ResultSetFactory.PICTURE_TYPE, resultSetAlias) + + override def createResultSetWriter[M <: MetaData, R <: Record](resultSet: ResultSet[_ <: MetaData, _ <: Record], + resultSetAlias: String): ResultSetWriter[M, R] = + engineExecutorContext.createResultSetWriter(resultSet, resultSetAlias).asInstanceOf[ResultSetWriter[M, R]] + + override def getResultSetReader[M <: MetaData, R <: Record](fsPath: FsPath): ResultSetReader[M, R] = + ResultSetReader.getResultSetReader(fsPath.getSchemaPath).asInstanceOf[ResultSetReader[M, R]] + + private def createResultSetWriter[M <: MetaData, R <: Record](resultSetType: String, resultSetAlias: String): ResultSetWriter[M, R] = + engineExecutorContext.createResultSetWriter(resultSetType, resultSetAlias).asInstanceOf[ResultSetWriter[M, R]] + + override def sendResultSet(resultSetWriter: ResultSetWriter[_ <: MetaData, _ <: Record]): Unit = { + engineExecutorContext.sendResultSet(resultSetWriter) + } + + override def getGatewayUrl: String = { + val instances = Utils.tryThrow { + Sender.getInstances(AppConnEngineConnConfiguration.GATEWAY_SPRING_APPLICATION.getValue) + } { t => new AppConnExecutionErrorException(75538, "获取gateway的url失败", t) } + if (instances.length == 0) throw new AppConnExecutionErrorException(75538, "获取gateway的url失败") + instances(0).getInstance + } + + override def fetchLinkisJobResultSetPaths(jobId: Long): Array[FsPath] = { + val task = fetchLinkisJob(jobId) + val resultSetLocation = task.getResultLocation + val user = task.getExecuteUser + FSFactory.getFsByProxyUser(new FsPath(resultSetLocation), user) match { + case fileSystem: FileSystem => + fileSystem.init(new util.HashMap[String, String]) + Utils.tryFinally { + import scala.collection.JavaConverters._ + fileSystem.listPathWithError(new FsPath(resultSetLocation)).getFsPaths.asScala.toArray[FsPath] + }(Utils.tryQuietly(fileSystem.close())) + } + } +} \ No newline at end of file diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnComputationExecutorManagerImpl.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnComputationExecutorManagerImpl.scala new file mode 100644 index 000000000..65453762c --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnComputationExecutorManagerImpl.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn.executor + +import org.apache.linkis.engineconn.computation.executor.creation.ComputationExecutorManagerImpl +import org.apache.linkis.manager.label.entity.Label + + + +class AppConnComputationExecutorManagerImpl extends ComputationExecutorManagerImpl { + + override protected def getLabelKey(labels: Array[Label[_]]): String = "appconn" +} diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnEngineConnExecutor.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnEngineConnExecutor.scala new file mode 100644 index 000000000..ebe3a0f5c --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AppConnEngineConnExecutor.scala @@ -0,0 +1,260 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn.executor + +import java.util +import java.util.concurrent.{ConcurrentHashMap, TimeUnit} + +import com.webank.wedatasphere.dss.appconn.core.AppConn +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager +import com.webank.wedatasphere.dss.common.label.{DSSLabel, EnvDSSLabel, LabelKeyConvertor} +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import com.webank.wedatasphere.dss.standard.app.development.listener.common.AbstractRefExecutionAction +import com.webank.wedatasphere.dss.standard.app.development.listener.core.Killable +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.{AsyncExecutionResponseRef, ExecutionResponseRef} +import com.webank.wedatasphere.dss.standard.app.sso.Workspace +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef +import org.apache.commons.lang.StringUtils +import org.apache.linkis.common.utils.{OverloadUtils, Utils} +import org.apache.linkis.engineconn.computation.executor.async.AsyncConcurrentComputationExecutor +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext +import org.apache.linkis.engineconn.launch.EngineConnServer +import org.apache.linkis.governance.common.utils.GovernanceConstant +import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, LoadResource, NodeResource} +import org.apache.linkis.manager.engineplugin.appconn.conf.AppConnEngineConnConfiguration +import org.apache.linkis.manager.engineplugin.appconn.exception.AppConnExecuteFailedException +import org.apache.linkis.manager.engineplugin.appconn.executor.AppConnEngineConnExecutor._ +import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.protocol.engine.JobProgressInfo +import org.apache.linkis.scheduler.executer.{AsynReturnExecuteResponse, ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse} +import org.apache.linkis.server.BDPJettyServerHelper + +import scala.collection.JavaConverters._ + +class AppConnEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) + extends AsyncConcurrentComputationExecutor { + + private var user: String = _ + + private val executorLabels: util.List[Label[_]] = new util.ArrayList[Label[_]](2) + + private val taskHashMap: ConcurrentHashMap[String, AsyncExecutionResponseRef] = new ConcurrentHashMap[String, AsyncExecutionResponseRef](8) + + //定时清除map元素 + Utils.defaultScheduler.scheduleAtFixedRate(new Runnable { + override def run(): Unit = Utils.tryAndError { + val iterator = taskHashMap.entrySet().asScala.filter(_.getValue.isCompleted).map(_.getKey).clone() + iterator.foreach(taskHashMap.remove(_)) + info(s"Cleaned ${iterator.size} history jobs, now only ${taskHashMap.size()} remained.") + } + }, 0, 5, TimeUnit.MINUTES) + + def setUser(user: String): Unit = this.user = user + + override def executeLine(engineExecutorContext: EngineExecutionContext, code: String): ExecuteResponse = { + info(s"The execution code is: $code, runtime properties is: ${BDPJettyServerHelper.gson.toJson(engineExecutorContext.getProperties)}.") + + val source = engineExecutorContext.getProperties.get(GovernanceConstant.TASK_SOURCE_MAP_KEY) match { + case map: util.Map[String, Object] => map + case _ => return ErrorExecuteResponse("Cannot find source.", null) + } + def getValue(map: util.Map[String, AnyRef], key: String): String = map.get(key) match { + case string: String => string + case anyRef: AnyRef => BDPJettyServerHelper.gson.toJson(anyRef) + case _ => null + } + + val workspace = BDPJettyServerHelper.gson.fromJson(getValue(engineExecutorContext.getProperties, WORKSPACE_NAME_STR), classOf[Workspace]) + val appConnName = getAppConnName(getValue(engineExecutorContext.getProperties, NODE_TYPE)) + val appConn = AppConnManager.getAppConnManager.getAppConn(appConnName) + if (appConn == null) { + error(s"Cannot find AppConn $appConnName.") + throw AppConnExecuteFailedException(510001, "Cannot Find appConnName: " + appConnName) + } + val labels = engineExecutorContext.getProperties.get("labels").toString + getAppInstanceByLabels(labels, appConn) match { + case Some(appInstance) => + val developmentIntegrationStandard = appConn.asInstanceOf[OnlyDevelopmentAppConn].getOrCreateDevelopmentStandard + val refExecutionService = developmentIntegrationStandard.getRefExecutionService(appInstance) + val refJobContent = if (StringUtils.isNotBlank(code)) BDPJettyServerHelper.gson.fromJson(code, classOf[util.HashMap[String, AnyRef]]) + else engineExecutorContext.getProperties + var submitUser = getValue(engineExecutorContext.getProperties, SUBMIT_USER_KEY) + if (submitUser == null) submitUser = user + val variables = engineExecutorContext.getProperties.get(VARIABLES_KEY) match { + case map: util.Map[String, Object] => map + case _ => new util.HashMap[String, Object]() + } + val responseRef = Utils.tryCatch { + AppConnExecutionUtils.tryToOperation(refExecutionService, getValue(engineExecutorContext.getProperties, CONTEXT_ID_KEY), + getValue(source, PROJECT_NAME_STR), new ExecutionRequestRefContextImpl(engineExecutorContext, user, submitUser), + getLabels(labels), getValue(source, NODE_NAME_STR), getValue(engineExecutorContext.getProperties, NODE_TYPE), + user, workspace, refJobContent, variables) + } (t => ExecutionResponseRef.newBuilder.setException(t).error()) + responseRef match { + case asyncResponseRef: AsyncExecutionResponseRef => + engineExecutorContext.getJobId match { + case Some(id) => + info(s"add async responseRef to task map, the taskId is $id.") + taskHashMap.put(id, asyncResponseRef) + case _ => + } + new AsynReturnExecuteResponse { + private var er: ExecuteResponse => Unit = _ + + def tryToNotifyAll(responseRef: ResponseRef): Unit = { + val executeResponse = createExecuteResponse(responseRef, appConnName) + if (er == null) this synchronized { + while (er == null) this.wait(1000) + } + er(executeResponse) + } + + override def notify(rs: ExecuteResponse => Unit): Unit = { + er = rs + this synchronized notifyAll() + } + + asyncResponseRef.notifyMe(new java.util.function.Consumer[ResponseRef] { + override def accept(t: ResponseRef): Unit = tryToNotifyAll(t) + }) + } + case responseRef: ResponseRef => + createExecuteResponse(responseRef, appConnName) + } + case None => + throw AppConnExecuteFailedException(510000, "Cannot Find AppInstance by labels." + labels) + } + } + + + private def createExecuteResponse(responseRef: ResponseRef, appConnName: String): ExecuteResponse = + if (responseRef.isSucceed) SuccessExecuteResponse() + else { + val exception = responseRef match { + case response: ExecutionResponseRef => response.getException + case _ => null + } + error(s"$appConnName execute failed, failed reason is ${responseRef.getErrorMsg}.", exception) + ErrorExecuteResponse(responseRef.getErrorMsg, exception) + } + + private def getAppConnName(nodeType: String) = { + StringUtils.split(nodeType, ".")(0) + } + + private def getLabels(labels: String): util.List[DSSLabel] = { + val envLabelValue = if (labels.contains(LabelKeyConvertor.ROUTE_LABEL_KEY) || labels.contains(EnvDSSLabel.DSS_ENV_LABEL_KEY) ) { + val labelMap = DSSCommonUtils.COMMON_GSON.fromJson(labels, classOf[util.Map[String, String]]) + labelMap.getOrDefault(LabelKeyConvertor.ROUTE_LABEL_KEY,labelMap.getOrDefault(EnvDSSLabel.DSS_ENV_LABEL_KEY, labels)) + } else labels + util.Arrays.asList(new EnvDSSLabel(envLabelValue)) + } + + private def getAppInstanceByLabels(labels: String, appConn: AppConn): Option[AppInstance] = { + val appInstanceList = appConn.getAppDesc.getAppInstancesByLabels(getLabels(labels)) + if (appInstanceList != null && appInstanceList.size() > 0) { + return Some(appInstanceList.get(0)) + } + None + } + + override def executeCompletely(engineExecutorContext: EngineExecutionContext, code: String, completedLine: String): ExecuteResponse = null + + override def progress(taskID: String): Float = 0 + + override def getProgressInfo(taskID: String): Array[JobProgressInfo] = Array.empty + + override def supportCallBackLogs(): Boolean = false + + override def getExecutorLabels(): util.List[Label[_]] = executorLabels + + override def setExecutorLabels(labels: util.List[Label[_]]): Unit = { + if (null != labels && !labels.isEmpty) { + executorLabels.clear() + executorLabels.addAll(labels) + } + } + + override def requestExpectedResource(expectedResource: NodeResource): NodeResource = null + + override def getCurrentNodeResource(): NodeResource = { + val properties = EngineConnServer.getEngineCreationContext.getOptions + if (properties.containsKey(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)) { + val settingClientMemory = properties.get(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key) + if (!settingClientMemory.toLowerCase().endsWith("g")) { + properties.put(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key, settingClientMemory + "g") + } + } + val resource = new CommonNodeResource + val usedResource = new LoadResource(OverloadUtils.getProcessMaxMemory, 1) + resource.setUsedResource(usedResource) + resource + } + + override def getId(): String = "AppConnEngineExecutor_" + id + + override def getConcurrentLimit: Int = AppConnEngineConnConfiguration.CONCURRENT_LIMIT.getValue + + override def killAll(): Unit = { + } + + override def killTask(taskID: String): Unit = { + warn(s"AppConn want to kill job: $taskID.") + if (taskHashMap.containsKey(taskID)) { + val response = taskHashMap.get(taskID) + response.getAction match { + case action: AbstractRefExecutionAction => + action.setKilledFlag(true) + case _ => + } + taskHashMap.remove(taskID) + response.getRefExecutionOperation match { + case killable: Killable => + killable.kill(response.getAction) + case _ => + } + info(s"AppConn Kill job: $taskID succeed.") + } else { + warn(s"AppConn Kill job: $taskID failed for task is not exist.") + } + super.killTask(taskID) + } +} + + +object AppConnEngineConnExecutor { + + private val WORKSPACE_NAME_STR = "workspace" + + private val PROJECT_NAME_STR = "projectName" + + private val FLOW_NAME_STR = "flowName" + + private val NODE_NAME_STR = "nodeName" + + private val NODE_TYPE = "nodeType" + + private val CONTEXT_ID_KEY = "contextID" + + private val SUBMIT_USER_KEY = "wds.dss.workflow.submit.user" + + private val VARIABLES_KEY = "variables" + +} \ No newline at end of file diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/ExecutionRequestRefContextImpl.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/ExecutionRequestRefContextImpl.scala new file mode 100644 index 000000000..f6069add2 --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/ExecutionRequestRefContextImpl.scala @@ -0,0 +1,86 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn.executor + +import java.util + +import com.webank.wedatasphere.dss.standard.app.development.listener.conf.RefExecutionConfiguration +import com.webank.wedatasphere.dss.standard.app.development.listener.core.LinkisJob +import com.webank.wedatasphere.dss.standard.app.development.listener.exception.AppConnExecutionErrorException +import org.apache.linkis.common.exception.ErrorException +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext +import org.apache.linkis.governance.common.entity.job.JobRequestWithDetail +import org.apache.linkis.governance.common.protocol.job.{JobReqQuery, JobRespProtocol} +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.rpc.Sender + +import scala.collection.JavaConversions._ + +/** + * Only jobhistory of linkis1.0 is applicable. + */ +class ExecutionRequestRefContextImpl(engineExecutorContext: EngineExecutionContext, user: String, submitUser: String) + extends AbstractExecutionRequestRefContext(engineExecutorContext, user, submitUser) { + + private val jobHistorySender:Sender = Sender.getSender(RefExecutionConfiguration.JOB_HISTORY_APPLICATION_NAME.getValue) + + override def fetchLinkisJob(jobId: Long): LinkisJob = { + val jobRequest:JobRequestWithDetail= new JobRequestWithDetail() + jobRequest.setId(jobId) + jobRequest.setSource(null) + val requestQueryTask = JobReqQuery(jobRequest) + val linkisJob:JobRequestWithDetail = Utils.tryThrow( + jobHistorySender.ask(requestQueryTask) match { + case responsePersist: JobRespProtocol => + val status = responsePersist.getStatus + if (status != 0){ + error(s"Fetch linkisJob from jobHistory failed, errorMsg: ${responsePersist.getMsg}.") + throw new AppConnExecutionErrorException(95541, s"Fetch linkisJob from jobHistory failed, errorMsg: ${responsePersist.getMsg}.") + } else { + val data = responsePersist.getData + data.get("jobHistoryList") match { + case tasks: util.List[JobRequestWithDetail] => + if (tasks.size() > 0) tasks.get(0) else throw new AppConnExecutionErrorException(95542, s"query from jobhistory not a correct List type taskId is $jobId") + + + case _ => throw new AppConnExecutionErrorException(95541, s"query from jobhistory not a correct List type taskId is $jobId") + } + } + + + case r => + error(s"Fetch linkisJob from jobHistory incorrectly, response is $r.") + throw new AppConnExecutionErrorException(95541, s"Fetch linkisJob from jobHistory incorrectly, response is $r.") + } + ){ + case errorException: ErrorException => errorException + case e: Exception => + new AppConnExecutionErrorException(95541, s"query taskId $jobId error.", e) + } + new LinkisJob { + override def getResultLocation: String = linkisJob.getSubJobDetailList.find(subJob =>{null != subJob.getResultLocation}).get.getResultLocation + override def getSubmitUser: String = linkisJob.getSubmitUser + override def getExecuteUser: String = linkisJob.getSubmitUser + override def getStatus: String = linkisJob.getStatus + override def getSource: util.Map[String, String] = linkisJob.getSource + override def getParams: util.Map[String, AnyRef] = linkisJob.getParams + override def getLabels: util.List[Label[_]] = linkisJob.getLabels + } + } + +} diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/factory/AppConnEngineConnFactory.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/factory/AppConnEngineConnFactory.scala new file mode 100644 index 000000000..cf868c409 --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/factory/AppConnEngineConnFactory.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn.factory + +import java.io.File + +import com.webank.wedatasphere.dss.appconn.loader.utils.AppConnUtils +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.engineconn.common.conf.EngineConnConf +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.common.engineconn.EngineConn +import org.apache.linkis.engineconn.computation.executor.creation.ComputationSingleExecutorEngineConnFactory +import org.apache.linkis.engineconn.executor.entity.LabelExecutor +import org.apache.linkis.governance.common.exception.engineconn.EngineConnExecutorErrorCode +import org.apache.linkis.manager.engineplugin.appconn.conf.AppConnEngineConnConfiguration +import org.apache.linkis.manager.engineplugin.appconn.executor.AppConnEngineConnExecutor +import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException +import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.manager.label.entity.engine.{EngineType, RunType} + +class AppConnEngineConnFactory extends ComputationSingleExecutorEngineConnFactory { + + private val appConnHomePath = new File(EngineConnConf.getWorkHome, AppConnUtils.APPCONN_DIR_NAME) + if (!appConnHomePath.exists() && !appConnHomePath.mkdir()) { + throw new EngineConnPluginErrorException(EngineConnExecutorErrorCode.INIT_EXECUTOR_FAILED, + s"Cannot mkdir ${appConnHomePath.getPath}, please make sure the permission is ok.") + } + DataWorkCloudApplication.setProperty(AppConnUtils.APPCONN_HOME_PATH.key, appConnHomePath.getPath) + warn(s"Set ${AppConnUtils.APPCONN_HOME_PATH.key}=${appConnHomePath.getPath}.") + + override protected def newExecutor(id: Int, + engineCreationContext: EngineCreationContext, + engineConn: EngineConn): LabelExecutor = { + /** + * This is for loading all AppConns when EngineConn is starting, 2 reasons: + * 1. Load AppConns will cost lots of time, since all zips must pull from BML + * 2. when a task is killed by user, and the AppConns is loading, the loading will be failed, cause a bug when new task submitted in later + */ + AppConnManager.getAppConnManager.init() + val executor = new AppConnEngineConnExecutor(AppConnEngineConnConfiguration.ENGINE_DEFAULT_LIMIT.getValue, id) + executor.setUser(engineCreationContext.getUser) + executor + } + + override protected def getEngineConnType: EngineType = EngineType.APPCONN + + override protected def getRunType: RunType = RunType.APPCONN + +} \ No newline at end of file diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/launch/AppConnProcessEngineConnLaunchBuilder.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/launch/AppConnProcessEngineConnLaunchBuilder.scala new file mode 100644 index 000000000..b1e09691a --- /dev/null +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/launch/AppConnProcessEngineConnLaunchBuilder.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.manager.engineplugin.appconn.launch + +import java.util + +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager +import com.webank.wedatasphere.dss.appconn.manager.impl.AbstractAppConnManager +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnInfoService +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.manager.common.protocol.bml.BmlResource +import org.apache.linkis.manager.common.protocol.bml.BmlResource.BmlResourceVisibility +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest +import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder + +import scala.collection.convert.wrapAsScala._ +import scala.collection.convert.wrapAsJava._ + +class AppConnProcessEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder{ + + override protected def getBmlResources(implicit engineConnBuildRequest: EngineConnBuildRequest): util.List[BmlResource] = { + super.getBmlResources + } + +} diff --git a/dss-appconn/pom.xml b/dss-appconn/pom.xml new file mode 100644 index 000000000..e93fbe43f --- /dev/null +++ b/dss-appconn/pom.xml @@ -0,0 +1,48 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../pom.xml + + 4.0.0 + + dss-appconn + pom + + + dss-appconn-core + dss-appconn-loader + dss-appconn-manager + dss-scheduler-appconn + linkis-appconn-engineplugin + appconns/dss-datachecker-appconn + appconns/dss-eventchecker-appconn + appconns/dss-schedulis-appconn + appconns/dss-workflow-appconn + appconns/dss-sendemail-appconn + appconns/dss-dolphinscheduler-appconn + appconns/dss-sso-appconn + appconns/dss-scriptis-appconn + + + \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/pom.xml b/dss-apps/dss-apiservice-server/pom.xml new file mode 100644 index 000000000..23b768776 --- /dev/null +++ b/dss-apps/dss-apiservice-server/pom.xml @@ -0,0 +1,424 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../pom.xml + + 4.0.0 + + dss-apiservice-server + + + + UTF-8 + 2.30.1 + 2.22.2 + 2.22.2 + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + asm + org.ow2.asm + + + hk2-api + org.glassfish.hk2 + + + jersey-common + org.glassfish.jersey.core + + + linkis-common + org.apache.linkis + + + xstream + com.thoughtworks.xstream + + + + + com.alibaba + fastjson + 1.2.83 + + + + org.glassfish.jersey.ext + jersey-bean-validation + ${jersey.version} + provided + + + javax.ws.rs-api + javax.ws.rs + + + hk2-locator + org.glassfish.hk2 + + + hk2-api + org.glassfish.hk2 + + + hibernate-validator + org.hibernate + + + jersey-server + org.glassfish.jersey.core + + + + + + org.apache.linkis + linkis-bml-client + ${linkis.version} + + + commons-beanutils + commons-beanutils + + + linkis-common + org.apache.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + org.apache.linkis + linkis-computation-client + ${linkis.version} + + + commons-beanutils + commons-beanutils + + + linkis-common + org.apache.linkis + + + + + org.postgresql + postgresql + 42.3.8 + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + org.apache.linkis + linkis-storage + ${linkis.version} + provided + + + linkis-common + org.apache.linkis + + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + archaius-core + com.netflix.archaius + + + slf4j-api + org.slf4j + + + spring-cloud-starter + org.springframework.cloud + + + spring-web + org.springframework + + + + + com.zaxxer + HikariCP + 3.4.1 + + + slf4j-api + org.slf4j + + + + + + + + + + + hk2-api + org.glassfish.hk2 + 2.4.0-b34 + + + + + + + + + org.modelmapper + modelmapper + 0.7.5 + + + + io.jsonwebtoken + jjwt + 0.6.0 + + + jackson-databind + com.fasterxml.jackson.core + + + + + + org.hamcrest + hamcrest-all + 1.3 + test + + + + + com.h2database + h2 + 2.1.210 + test + + + + com.ninja-squad + DbSetup + 2.1.0 + test + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + test + + + spring-boot-autoconfigure + org.springframework.boot + + + + + + junit + junit + 4.12 + test + + + org.junit.platform + junit-platform-launcher + 1.5.2 + test + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + xstream + com.thoughtworks.xstream + 1.4.19 + + + + + + + + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + maven-failsafe-plugin + ${maven-failsafe-plugin.version} + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + + + + + + + + + + ${project.artifactId}-${project.version} + + diff --git a/dss-apps/dss-apiservice-server/src/main/assembly/distribution.xml b/dss-apps/dss-apiservice-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..9882c22ec --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/assembly/distribution.xml @@ -0,0 +1,44 @@ + + + + dss-apiService-server + + dir + + true + dss-apiservice-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ApiServiceGetAction.scala b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ApiServiceGetAction.scala new file mode 100644 index 000000000..47a97be54 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ApiServiceGetAction.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.action + +import org.apache.linkis.httpclient.request.GetAction +import org.apache.linkis.ujes.client.request.UJESJobAction + +class ApiServiceGetAction extends GetAction with UJESJobAction { + override def suffixURLs: Array[String] = Array("dss","framework","workspace", "getWorkspaceIdByUserName") +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultSetDownloadAction.scala b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultSetDownloadAction.scala new file mode 100644 index 000000000..02f4b2bdd --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultSetDownloadAction.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.action + +import org.apache.linkis.httpclient.request.{DownloadAction, GetAction} +import org.apache.linkis.ujes.client.request.UJESJobAction +import org.apache.http.HttpResponse + +import scala.tools.nsc.interpreter.InputStream + + + +class ResultSetDownloadAction extends GetAction with DownloadAction with UJESJobAction { + + private var inputStream: InputStream = _ + + private var response: HttpResponse = _ + + override def write(inputStream: InputStream): Unit = this.inputStream = inputStream + + def getInputStream: InputStream = inputStream + + override def suffixURLs: Array[String] = Array("filesystem", "resultsetToExcel") + + + def getResponse: HttpResponse = response + + def setResponse(response: HttpResponse): Unit = this.response = response +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultWorkspaceIds.scala b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultWorkspaceIds.scala new file mode 100644 index 000000000..9d583ab8d --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultWorkspaceIds.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.action + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/framework/workspace/getWorkspaceIdByUserName") +class ResultWorkspaceIds extends DWSResult{ + var userWorkspaceIds:String = _ + def getUserWorkspaceIds:String = this.userWorkspaceIds + def setUserWorkspaceIds(userWorkspaceIds:String):Unit = this.userWorkspaceIds = userWorkspaceIds +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiCommentUpdateRequest.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiCommentUpdateRequest.java new file mode 100644 index 000000000..f8d111a01 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiCommentUpdateRequest.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.bo; + +public class ApiCommentUpdateRequest { + private Long id; + private String comment; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceJob.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceJob.java new file mode 100644 index 000000000..f7ba5acfb --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceJob.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + +import org.apache.linkis.ujes.client.response.JobExecuteResult; + + +public class ApiServiceJob { + + private String submitUser; + private String proxyUser; + private JobExecuteResult jobExecuteResult; + public String getSubmitUser() { + return submitUser; + } + + public void setSubmitUser(String submitUser) { + this.submitUser = submitUser; + } + + + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + public JobExecuteResult getJobExecuteResult() { + return jobExecuteResult; + } + + public void setJobExecuteResult(JobExecuteResult jobExecuteResult) { + this.jobExecuteResult = jobExecuteResult; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceQuery.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceQuery.java new file mode 100644 index 000000000..1f6035b55 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceQuery.java @@ -0,0 +1,122 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + + + +public class ApiServiceQuery { + + /** + * 服务名称 + */ + private String name; + + /** + * 标签 + */ + private String tag; + + /** + * 状态 + */ + private Integer status; + + /** + * api创建者 + */ + private String creator; + + private int currentPage; + + private int pageSize; + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + private int workspaceId; + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + private String userName; + + public ApiServiceQuery(String userName,String name, String tag, Integer status, String creator) { + this.userName= userName; + this.name = name; + this.tag = tag; + this.status = status; + this.creator = creator; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public int getCurrentPage() { + return currentPage; + } + + public void setCurrentPage(int currentPage) { + this.currentPage = currentPage; + } + + public int getPageSize() { + return pageSize; + } + + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceToken.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceToken.java new file mode 100644 index 000000000..fb2b1b031 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceToken.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + + + +import java.util.Date; + + + +public class ApiServiceToken { + + private String applyUser; + private Date applyTime; + private Long apiServiceId; + private String publisher; + + public String getPublisher() { + return publisher; + } + + public void setPublisher(String publisher) { + this.publisher = publisher; + } + + public Long getApiServiceId() { + return apiServiceId; + } + + public void setApiServiceId(Long apiServiceId) { + this.apiServiceId = apiServiceId; + } + + public Date getApplyTime() { + return applyTime; + } + + public void setApplyTime(Date applyTime) { + this.applyTime = applyTime; + } + + public String getApplyUser() { + return applyUser; + } + + public void setApplyUser(String applyUser) { + this.applyUser = applyUser; + } + + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/DirFileTree.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/DirFileTree.java new file mode 100644 index 000000000..30005e251 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/DirFileTree.java @@ -0,0 +1,79 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + +import java.util.HashMap; +import java.util.List; + + +public class DirFileTree { + + private String name; + private String path; + private HashMap properties; + private List children; + private Boolean isLeaf = false; + private String parentPath; + + public Boolean getIsLeaf() { + return isLeaf; + } + + public void setIsLeaf(Boolean isLeaf) { + this.isLeaf = isLeaf; + } + + public String getParentPath() { + return parentPath; + } + + public void setParentPath(String parentPath) { + this.parentPath = parentPath; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public HashMap getProperties() { + return properties; + } + + public void setProperties(HashMap properties) { + this.properties = properties; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/LinkisExecuteResult.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/LinkisExecuteResult.java new file mode 100644 index 000000000..2314e9bfd --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/LinkisExecuteResult.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + + +public class LinkisExecuteResult { + + public LinkisExecuteResult(String taskId,String execId){ + this.taskId = taskId; + this.execId = execId; + } + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + private String taskId; + + public String getExecId() { + return execId; + } + + public void setExecId(String execId) { + this.execId = execId; + } + + private String execId; +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/QueryRequest.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/QueryRequest.java new file mode 100644 index 000000000..87bec95d0 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/QueryRequest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.bo; + +import java.util.Map; + + +public class QueryRequest { + /** + * 模块名,也是登录用户名、密码 + */ + private String moduleName; + + /** + * 调用参数 + */ + private Map params; + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + public String getModuleName() { + return moduleName; + } + + public void setModuleName(String moduleName) { + this.moduleName = moduleName; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/TokenQuery.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/TokenQuery.java new file mode 100644 index 000000000..6cede5247 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/TokenQuery.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + + +import java.util.Date; + +public class TokenQuery { + private Long apiId; + private String user; + private Integer status; + + private String startDate; + private String endDate; + + private int currentPage; + + private int pageSize; + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + private String creator; + + + public TokenQuery(Long apiId, String user, Integer status) { + this.apiId = apiId; + this.user = user; + this.status = status; + } + + public TokenQuery(Long apiId, String user, Integer status, String startDate, String endDate) { + this.apiId = apiId; + this.user = user; + this.status = status; + this.startDate = startDate; + this.endDate = endDate; + } + + public TokenQuery() { + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } + + public int getCurrentPage() { + return currentPage; + } + + public void setCurrentPage(int currentPage) { + this.currentPage = currentPage; + } + + public int getPageSize() { + return pageSize; + } + + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } + + public Long getApiId() { + return apiId; + } + + public void setApiId(Long apiId) { + this.apiId = apiId; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/VariableString.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/VariableString.java new file mode 100644 index 000000000..af901365c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/VariableString.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.bo; + + +public class VariableString { + private String path; + + public VariableString(String unparsedPath) { + this.path = unparsedPath; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java new file mode 100644 index 000000000..36244363f --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.config; + +import org.apache.linkis.common.conf.CommonVars; + + +public class ApiServiceConfiguration { + public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.api.service.author.user.token","WS-AUTH"); + public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.api.service.adminuser","ws"); + + public final static CommonVars LINKIS_CONNECTION_TIMEOUT = CommonVars.apply("wds.linkis.flow.connection.timeout",30000); + public final static CommonVars LINKIS_API_VERSION = CommonVars.apply("wds.linkis.server.version","v1"); + + public final static CommonVars API_SERVICE_TOKEN_KEY = CommonVars.apply("wds.dss.api.service.token.key","ApiServiceToken"); + + + public final static CommonVars DSS_API_TOKEN_SECRET_ID = CommonVars.apply("wds.dss.api.service.secret", "DSSSECRETTEST001002"); + + public final static CommonVars LINKIS_JOB_REQUEST_STATUS_TIME = CommonVars.apply("wds.linkis.job.status.timeout",3000); + + public final static CommonVars LOG_ARRAY_LEN = CommonVars.apply("wds.linkis.log.array.len",4); + + public final static CommonVars RESULT_PRINT_SIZE = CommonVars.apply("wds.linkis.result.print.size",10); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApiCommonConstant.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApiCommonConstant.java new file mode 100644 index 000000000..a91850d29 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApiCommonConstant.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.constant; + + +public class ApiCommonConstant { + public static final String DEFAULT_APPROVAL_NO="0001"; + public static final Integer API_DELETE_STATUS=2; +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApprovalStatus.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApprovalStatus.java new file mode 100644 index 000000000..12a5eac87 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApprovalStatus.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.constant; + +public enum ApprovalStatus { + INIT("init", 0), + APPLYING("apply", 1), + SUCCESS("success", 2), + FAIELD("failed", 3); + + String key; + int value; + + ApprovalStatus(String key, int value) { + this.key = key; + this.value = value; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public int getValue() { + return value; + } + + public void setValue(int value) { + this.value = value; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamType.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamType.java new file mode 100644 index 000000000..56691675c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamType.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.constant; + + +public class ParamType { + public static final String number = "2"; + + public static final String array = "4"; + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamTypeEnum.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamTypeEnum.java new file mode 100644 index 000000000..239469eef --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamTypeEnum.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.constant; + + +public enum ParamTypeEnum { + + STRING(1, "String"), + + NUMBER(2, "Number"), + + DATE(3, "Date"), + + ARRAY(4, "Array"); + + private Integer index; + private String name; + + ParamTypeEnum(Integer index, String name) { + this.index = index; + this.name = name; + } + + public static ParamTypeEnum getEnum(Integer index) { + if (index == null) { + return STRING; + } + for (ParamTypeEnum statusEnum : values()) { + if (statusEnum.getIndex().equals(index)) { + return statusEnum; + } + } + return STRING; + } + + public Integer getIndex() { + return this.index; + } + + public String getName() { + return this.name; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/RequireEnum.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/RequireEnum.java new file mode 100644 index 000000000..a96906f00 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/RequireEnum.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.constant; + + +public enum RequireEnum { + + YES(1, "是"), + + NO(0, "否"); + + private Integer index; + private String name; + + RequireEnum(Integer index, String name) { + this.index = index; + this.name = name; + } + + public static RequireEnum getEnum(Integer index) { + if (index == null) { + return NO; + } + for (RequireEnum statusEnum : values()) { + if (statusEnum.getIndex().equals(index)) { + return statusEnum; + } + } + return NO; + } + + public Integer getIndex() { + return this.index; + } + + public String getName() { + return this.name; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SQLMetadataInfoCheckStatus.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SQLMetadataInfoCheckStatus.java new file mode 100644 index 000000000..15de7e2e5 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SQLMetadataInfoCheckStatus.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.constant; + + + + +public enum SQLMetadataInfoCheckStatus { + LEGAL(0, "该SQL查询合法"), ILLEGALSQL(1,"不支持无库表信息的SQL查询"); + + int index; + String desc; + + SQLMetadataInfoCheckStatus(int index, String desc) { + this.index = index; + this.desc = desc; + } + + public String getDescByIndex(int index) { + for (SQLMetadataInfoCheckStatus item : values()) { + if (item.getIndex() == index) { + return item.getDesc(); + } + } + return null; + } + + public int getIndex() { + return index; + } + + public String getDesc() { + return desc; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SaveTokenEnum.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SaveTokenEnum.java new file mode 100644 index 000000000..8c6b5380c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SaveTokenEnum.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.constant; + + +public enum SaveTokenEnum { + SUCCESS(1, "success"), FAILED(2, "failed"), DUPLICATE(3, "duplicate"); + + int index; + String value; + SaveTokenEnum(int index, String value) { + this.index = index; + this.value = value; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/StatusEnum.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/StatusEnum.java new file mode 100644 index 000000000..c56cd6008 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/StatusEnum.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.constant; + + +public enum StatusEnum { + + DISABLE(0, "停止"), + + ENABLE(1, "运行中"); + + private Integer index; + private String name; + + StatusEnum(Integer index, String name) { + this.index = index; + this.name = name; + } + + public static StatusEnum getEnum(Integer index) { + if (index == null) { + return null; + } + for (StatusEnum statusEnum : values()) { + if (statusEnum.getIndex().equals(index)) { + return statusEnum; + } + } + return null; + } + + public Integer getIndex() { + return this.index; + } + + public String getName() { + return this.name; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceAccessDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceAccessDao.java new file mode 100644 index 000000000..2b8c1b15d --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceAccessDao.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiAccessVo; + + +public interface ApiServiceAccessDao { + void addAccessRecord(ApiAccessVo apiAccessVo); + + ApiAccessVo queryByVersionId(Long versionId); + + ApiAccessVo queryByApiId(Long apiServiceId); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceApprovalDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceApprovalDao.java new file mode 100644 index 000000000..2aab48a28 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceApprovalDao.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + + +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import org.apache.ibatis.annotations.Param; + +public interface ApiServiceApprovalDao { + + /** + * 新增一条审批单 + */ + void insert(ApprovalVo approvalVo); + + /** + * 查询审批单 + * */ + ApprovalVo queryByApprovalNo(@Param("approvalNo") String approvalNo); + + /** + * set status to success + * */ + void setApprovalStatusSuccess(@Param("approvalNo") String approvalNo); + + /** + * set status to failed + * */ + void setApprovalStatusFailed(@Param("approvalNo") String approvalNo); + + /** + * set status to init + * */ + void setApprovalStatusInit(@Param("approvalNo") String approvalNo); + + /** + * set status to applying + * */ + void setApprovalStatusApplying(@Param("approvalNo") String approvalNo); + + /** + * update status + * */ + void updateApprovalStatus(@Param("approvalNo") String approvalNo, @Param("status") Integer status); + + + /** + * delete approval + * */ + void deleteApproval(@Param("approvalNo") String approvalNo); + + /** + * 通过版本ID查询审批单 + * */ + ApprovalVo queryByVersionId(@Param("apiVersionId") Long apiVersionId); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceDao.java new file mode 100644 index 000000000..591bbf1a8 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceDao.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceQuery; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiServiceVo; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +public interface ApiServiceDao { + + /** + * Insert + */ + void insert(ApiServiceVo apiServiceVo); + + /** + * Update + */ + void update(ApiServiceVo apiServiceVo); + + + void updateToTarget(ApiServiceVo apiServiceVo); + + /** + * query + */ + List query(ApiServiceQuery query); + + /** + * query + */ + + List queryByScriptPath(@Param("scriptPath") String scriptPath); + + /** + * query + */ + ApiServiceVo queryByPath(String path); + + /** + * query api path count + */ + Integer queryCountByPath(@Param("scriptPath") String scriptPath, @Param("path") String path); + + /** + * query api name + */ + Integer queryCountByName(String name); + + Integer enableApi(@Param("id") Long id); + + Integer disableApi(@Param("id") Long id); + + Integer deleteApi(@Param("id") Long id); + + List queryAllTags(@Param("userName")String userName,@Param("workspaceId")Integer workspaceId); + + ApiServiceVo queryById(@Param("id") Long id); + + List queryByWorkspaceId(@Param("workspaceId") Integer workspaceId, @Param("userName") String userName); + + Integer updateApiServiceComment(@Param("id") Long id, @Param("comment") String comment); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceParamDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceParamDao.java new file mode 100644 index 000000000..918b7f40c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceParamDao.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + + +import com.webank.wedatasphere.dss.apiservice.core.vo.ParamVo; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +public interface ApiServiceParamDao { + + /** + * insert + * + * @param param + * @return + */ + int insert(ParamVo param); + + /** + * Update + * + * @param param + */ + void update(ParamVo param); + + /** + * query + */ +// List queryByConfigIdAndVersion(@Param("configId") Long configId, @Param("version") String version); + + List queryByVersionId(@Param("apiVersionId") Long apiVersionId); + + /** + * query a line by id + * @param paramId + * @return paramVo + * */ + int queryById(@Param("paramId") Long paramId); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceTokenManagerDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceTokenManagerDao.java new file mode 100644 index 000000000..eea29f3a6 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceTokenManagerDao.java @@ -0,0 +1,115 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import com.webank.wedatasphere.dss.apiservice.core.bo.TokenQuery; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +public interface ApiServiceTokenManagerDao { + + /** + * insert a token + * */ + void insert(TokenManagerVo tokenManagerVo); + + /** + * insert batch token + * */ + void insertList(List tokenManagerVos); + + + /** + * 多条记录需要去重返回最新授权的 + * @param userName + * @return + */ + List queryByApplyUser(String userName); + + + List queryByApplyUserAndServiceId(@Param("userName") String userName, @Param("serviceId") Long serviceId); + + List queryByApplyUserAndVersionId(@Param("userName") String userName, @Param("apiVersionId") Long apiVersionId); + + + + /** + * query + * */ + List query(TokenQuery tokenQuery); + + /** + * query token by token id + * */ + TokenManagerVo queryTokenById(@Param("id") Long tokenId); + + /** + * query token by api service id + * */ + List queryTokenByApiServiceId(@Param("api_id") Long apiServiceId); + + /** + * query token avoid submit again + * */ + int queryApprovalNo(@Param("approvalNo") String approvalNo); + + /** + * query tokens according to status + * */ + List queryTokenByStatus(@Param("status") Integer status); + + /** + * disable token status + * */ + void disableTokenStatus(@Param("id") Long id); + + + /** + * disable token status + * */ + void disableTokenStatusByVersionId(@Param("apiVersionId") Long apiVersionId); + + + /** + * disable token status + * */ + void disableTokenStatusByApiId(@Param("apiId") Long apiId); + + /** + * enable token status + * */ + void enableTokenStatus(@Param("id") Long id); + + void enableTokenStatusByVersionId(@Param("apiVersionId") Long apiVersionId); + + void enableTokenStatusByApiId(@Param("apiId") Long apiId); + + + /** + * delete token + * */ + void deleteTokenById(@Param("id") Long tokenId); + + /** + * update token status + * */ +// void updateTokenStatus(@Param(id) Long tokenId, String status); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceVersionDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceVersionDao.java new file mode 100644 index 000000000..521efd41f --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceVersionDao.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + + +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiVersionVo; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +public interface ApiServiceVersionDao { + + /** + * inster apiversionVo + * */ + void insert(ApiVersionVo apiVersionVo); + + /** + * query version + * */ + ApiVersionVo queryApiVersionByVersionId(@Param("id") Long apiVersionId); + + List queryApiVersionByApiServiceId(@Param("api_id") Long apiServiceId); + + /** + * update api version + * */ + void updateApiVersionStatusById(@Param("id") Long apiVersionId, @Param("status") int status); + + void updateApiVersionStatus(ApiVersionVo apiVersionVo); + + /** + * update all api version status by service id + * */ + void updateAllApiVersionStatusByApiServiceId(@Param("api_id") Long apiServiceId, @Param("status") int status); + + void disableApiVersionStatusByApiId(@Param("api_id") Long apiServiceId); + + /** + * delete api version + * */ + void deleteApiVersionById(@Param("id") Long apiVersionId); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceAccessMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceAccessMapper.xml new file mode 100644 index 000000000..ecda53d98 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceAccessMapper.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + `login_user`,`api_name`,`api_id`,`api_version_id`, `api_publisher`, `access_time`, `execute_user` + + + + `id`,`login_user`,`api_name`,`api_id`,`api_version_id`, `api_publisher`, `access_time`, `execute_user` + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApiVersionMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApiVersionMapper.xml new file mode 100644 index 000000000..48460f34b --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApiVersionMapper.xml @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + + + + + + + + `api_id`,`version`,`bml_resource_id`,`bml_version`,`source`,`creator`,`create_time`,`status`, + `metadata_info`, `auth_id` + + + + `id`,`api_id`,`version`,`bml_resource_id`,`bml_version`,`source`,`creator`,`create_time`,`status`, + `metadata_info`, `auth_id` + + + + + + + + + + + + + + UPDATE `dss_apiservice_api_version` + SET `status` = #{status} + WHERE id = #{id} + + + + UPDATE `dss_apiservice_api_version` + SET `status` = 0 + WHERE + api_id = #{api_id} + + + + UPDATE `dss_apiservice_api_version` + SET `status` = #{status} + WHERE id = #{id} + + + + UPDATE `dss_apiservice_api_version` + SET `status` = #{status} + WHERE api_id = #{api_id} + + + + + DELETE FROM + `dss_apiservice_api_version` + WHERE + id = #{id} + + + + \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApprovalMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApprovalMapper.xml new file mode 100644 index 000000000..95861ee90 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApprovalMapper.xml @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + `api_id`, `api_version_id`, `approval_name`, `apply_user`, `execute_user`, `creator`, + `status`, `create_time`, `update_time`, `approval_no` + + + + `id`, `api_version_id`, `api_id`, `approval_name`, `apply_user`, `execute_user`, `creator`, + `status`, `create_time`, `update_time`, `approval_no` + + + + + + + + + + + + UPDATE `dss_apiservice_approval` + SET `status` = 3 + WHERE `approval_no` = #{approvalNo} + + + + UPDATE `dss_apiservice_approval` + SET `status` = 4 + WHERE `approval_no` = #{approvalNo} + + + + UPDATE `dss_apiservice_approval` + SET `status` = 1 + WHERE `approval_no` = #{approvalNo} + + + + UPDATE `dss_apiservice_approval` + SET `status` = 2 + WHERE `approval_no` = #{approvalNo} + + + + UPDATE `dss_apiservice_approval` + SET `status` = #{status} + WHERE `approval_no` = #{approvalNo} + + + + DELETE FROM + `dss_apiservice_approval` + WHERE `approval_no` = #{approvalNo} + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceMapper.xml new file mode 100644 index 000000000..34d6ab302 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceMapper.xml @@ -0,0 +1,297 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `name`,`alias_name`, `path`,`protocol`,`method`,`tag`, `scope`,`description`,`status`,`type`,`run_type`,`create_time`, + `modify_time`,`creator`,`modifier`,`script_path`,`workspaceID`,`api_comment` + + + + `id`, `alias_name`, `name`,`path`,`protocol`,`method`, `tag`, `scope`,`description`,`status`,`type`,`run_type`,`create_time`, + `modify_time`,`creator`,`modifier`,`script_path`,`workspaceID`,`api_comment` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id` = #{id} + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id` = #{id} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceParamMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceParamMapper.xml new file mode 100644 index 000000000..e1b449032 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceParamMapper.xml @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + `api_version_id`,`name`,`type`,`required`,`description`, `display_name`, `default_value`, `details` + + + + `id`, `api_version_id`, `name`,`type`,`required`,`description`, `display_name`, `default_value`, `details` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id` = #{id} + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceTokenManagerMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceTokenManagerMapper.xml new file mode 100644 index 000000000..1ee6c7399 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceTokenManagerMapper.xml @@ -0,0 +1,245 @@ + + + + + + + + + + + + + + + + + + + + + + + + `api_id`,`api_version_id`,`publisher`,`user`,`apply_time`,`duration`, + `reason`,`ip_whitelist`,`status`,`caller`,`access_limit`,`token`,`apply_source` + + + + `id`, `api_version_id`,`api_id`,`publisher`,`user`,`apply_time`,`duration`, + `reason`,`ip_whitelist`,`status`,`caller`,`access_limit`,`token`,`apply_source` + + + + `id`, `api_version_id`,`api_id`,`publisher`,`user`,`apply_time`,`duration`, + `reason`,`ip_whitelist`,`status`,`caller`,`access_limit`,`apply_source` + + + + + + + + + + + + + + + (#{item.apiId},#{item.apiVersionId}, #{item.publisher}, #{item.user}, #{item.applyTime}, + #{item.duration}, #{item.reason}, #{item.ipWhitelist}, #{item.status}, + #{item.caller}, #{item.accessLimit}, #{item.token}, #{item.applySource}) + + + + + + + + + + + + + + + + + + + + + + + + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 1 + WHERE `id` = #{id} + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 0 + WHERE `id` = #{id} + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 0 + WHERE `api_version_id` = #{apiVersionId} + + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 1 + WHERE `api_version_id` = #{apiVersionId} + + + + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 0 + WHERE `api_id` = #{apiId} + + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 1 + WHERE `api_id` = #{apiId} + + + + DELETE FROM + `dss_apiservice_token_manager` + WHERE + id = #{id} + + + \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiExecuteException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiExecuteException.java new file mode 100644 index 000000000..f59092005 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiExecuteException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class ApiExecuteException extends ErrorException { + + public ApiExecuteException(int errCode, String desc) { + super(errCode, desc); + } + + public ApiExecuteException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + } diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceQueryException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceQueryException.java new file mode 100644 index 000000000..9fc2b0020 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceQueryException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class ApiServiceQueryException extends ErrorException { + + public ApiServiceQueryException(int errCode, String desc) { + super(errCode, desc); + } + + public ApiServiceQueryException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceRuntimeException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceRuntimeException.java new file mode 100644 index 000000000..4ac983f1c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceRuntimeException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.exception; + + +public class ApiServiceRuntimeException extends RuntimeException { + + public ApiServiceRuntimeException(String message) { + super(message); + } + + public ApiServiceRuntimeException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceTokenException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceTokenException.java new file mode 100644 index 000000000..50acd0979 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceTokenException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class ApiServiceTokenException extends ErrorException { + + public ApiServiceTokenException(int errCode, String desc) { + super(errCode, desc); + } + + public ApiServiceTokenException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + } diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/AssertException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/AssertException.java new file mode 100644 index 000000000..346b93722 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/AssertException.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.exception; + + +public class AssertException extends ApiServiceRuntimeException { + + public AssertException(String message) { + super(message); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/BeanValidationExceptionMapper.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/BeanValidationExceptionMapper.java new file mode 100644 index 000000000..0c1bc1061 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/BeanValidationExceptionMapper.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.exception; + +import com.webank.wedatasphere.dss.common.utils.MessageUtils; +import org.apache.linkis.server.Message; + +import javax.validation.ConstraintViolationException; +import javax.validation.ValidationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + + +@Provider +public class BeanValidationExceptionMapper implements ExceptionMapper { + @Override + public Response toResponse(ValidationException exception) { + StringBuilder stringBuilder = new StringBuilder(); + ((ConstraintViolationException)exception) + .getConstraintViolations().forEach(constraintViolation -> stringBuilder.append(constraintViolation.getMessage()).append(";")); + Message message = Message.error("Bean validation error[实例校验出错], detail:" + stringBuilder.toString()); + return MessageUtils.messageToResponse(message); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ApiServiceExecuteJob.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ApiServiceExecuteJob.java new file mode 100644 index 000000000..9fe4d7eda --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ApiServiceExecuteJob.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.execute; + +import org.apache.linkis.ujes.client.response.JobExecuteResult; + +import java.util.Map; + + +public interface ApiServiceExecuteJob { + String getCode(); + + void setCode(String code); + + String getEngineType(); + + void setEngineType(String engineType); + + String getRunType(); + + void setRunType(String runType); + + String getUser(); + + void setUser(String user); + + String getJobName(); + + Map getParams(); + + void setParams(Map params); + + Map getRuntimeParams(); + void setRuntimeParams(Map runtimeParams); + + JobExecuteResult getJobExecuteResult(); + + void setJobExecuteResult(JobExecuteResult jobExecuteResult); + + Map getJobProps(); + + void setJobProps(Map jobProps); + + String getScriptPath(); + void setScriptePath(String path); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/DefaultApiServiceJob.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/DefaultApiServiceJob.java new file mode 100644 index 000000000..0e1f2150d --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/DefaultApiServiceJob.java @@ -0,0 +1,141 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.execute; + +import org.apache.linkis.ujes.client.request.JobExecuteAction; +import org.apache.linkis.ujes.client.response.JobExecuteResult; + +import java.util.Map; + + +public class DefaultApiServiceJob implements ApiServiceExecuteJob { + private Map jobProps; + private String user; + private Map source; + private Map variables; + + private Map configuration; + + private String code; + + private String engineType; + + private String runType; + + private Map params; + + private Map runtimeParams; + + private JobExecuteResult jobExecuteResult; + + private String scriptPath; + + @Override + public String getCode() { + return this.code; + } + + @Override + public void setCode(String code) { + this.code = code; + } + + @Override + public String getEngineType() { + return engineType; + } + + @Override + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + @Override + public String getRunType() { + return runType; + } + + @Override + public void setRunType(String runType) { + this.runType = runType; + } + + @Override + public String getUser() { + return user; + } + + @Override + public void setUser(String user){ + this.user =user; + } + + @Override + public String getJobName() { + return null; + } + + @Override + public Map getParams() { + return params; + } + + @Override + public void setParams(Map params) { + this.params = params; + } + + @Override + public Map getRuntimeParams() { + return runtimeParams; + } + + @Override + public void setRuntimeParams(Map runtimeParams) { + this.runtimeParams = runtimeParams; + } + + @Override + public JobExecuteResult getJobExecuteResult() { + return this.jobExecuteResult; + } + + @Override + public void setJobExecuteResult(JobExecuteResult jobExecuteResult) { + this.jobExecuteResult = jobExecuteResult; + } + + @Override + public Map getJobProps() { + return jobProps; + } + + @Override + public void setJobProps(Map jobProps) { + this.jobProps = jobProps; + } + + @Override + public String getScriptPath() { + return scriptPath; + } + + @Override + public void setScriptePath(String path) { + this.scriptPath = path; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java new file mode 100644 index 000000000..19b030477 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java @@ -0,0 +1,257 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.execute; + +import com.webank.wedatasphere.dss.apiservice.core.action.ApiServiceGetAction; +import com.webank.wedatasphere.dss.apiservice.core.action.ResultSetDownloadAction; +import com.webank.wedatasphere.dss.apiservice.core.action.ResultWorkspaceIds; +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiExecuteException; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.request.ResultSetAction; +import org.apache.linkis.ujes.client.request.ResultSetListAction; +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.apache.linkis.ujes.client.response.JobInfoResult; +import org.apache.linkis.ujes.client.response.JobLogResult; +import org.apache.linkis.ujes.client.response.ResultSetListResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class ExecuteCodeHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExecuteCodeHelper.class); + + private static final String RELEASE_SCALA_TEMPLATE = "import org.apache.spark.sql.DataFrame\n" + + "val sql = %s\n" + + "val df = sqlContext.sql(sql)\n" + + "show(df)\n"; + + + private static final String SCALA_MARK = "\"\"\""; + + private static final String EXECUTE_SCALA_TEMPLATE = "import org.apache.spark.sql.DataFrame\n" + + "val executeCode = %s\n" + + "val df = sqlContext.sql(executeCode)\n" + + "show(df)\n"; + + + + public static String packageCodeToRelease(String executeCode){ + String retStr = String.format(RELEASE_SCALA_TEMPLATE, SCALA_MARK + executeCode + SCALA_MARK); + LOGGER.info("release scala code is {}", retStr); + return retStr; + } + + public static String packageCodeToExecute(String executeCode, String metaDataInfo){ + String retStr = String.format(EXECUTE_SCALA_TEMPLATE, SCALA_MARK + executeCode + SCALA_MARK); + LOGGER.info("execute scala code is {}", retStr); + return retStr; + } + + + public static Map getMetaDataInfoByExecute(String user, + String executeCode, + Map params, + String scriptPath) throws Exception { + Map resultMap = new HashMap<>(); + UJESClient client = LinkisJobSubmit.getClient(); + ApiServiceExecuteJob job = new DefaultApiServiceJob(); + //sql代码封装成scala执行 + job.setCode(ExecuteCodeHelper.packageCodeToRelease(executeCode)); + job.setEngineType("spark"); + job.setRunType("scala"); + job.setUser(user); + job.setParams(null); + // pattern注入 + job.setRuntimeParams((Map) params.get("variable")); + job.setScriptePath(scriptPath); + JobExecuteResult jobExecuteResult = LinkisJobSubmit.execute(job, client, "IDE"); + job.setJobExecuteResult(jobExecuteResult); + try { + waitForComplete(job, client); + } catch (Exception e) { + LOGGER.warn("Failed to execute job", e); + String reason = getLog(job, client); + LOGGER.error("Reason for failure: " + reason); + throw new ApiExecuteException(800024,"获取库表信息失败,执行脚本出错!"); + } + int resultSize = getResultSize(job, client); + for(int i =0; i < resultSize; i++){ + String result = getResult(job, i, ApiServiceConfiguration.RESULT_PRINT_SIZE.getValue(),client); + LOGGER.info("The content of the " + (i + 1) + "th resultset is :" + + result); + resultMap.put(Integer.toString(i),result); + + } + + LOGGER.info("Finished to execute job"); + return resultMap; + } + + + public static void waitForComplete(ApiServiceExecuteJob job, UJESClient client) throws Exception { + JobInfoResult jobInfo = client.getJobInfo(job.getJobExecuteResult()); + while (!jobInfo.isCompleted()) { + LOGGER.info("Update Progress info:" + getProgress(job, client)); + LOGGER.info("<----linkis log ---->"); + Utils.sleepQuietly(ApiServiceConfiguration.LINKIS_JOB_REQUEST_STATUS_TIME.getValue(job.getJobProps())); + jobInfo = client.getJobInfo(job.getJobExecuteResult()); + } + if (!jobInfo.isSucceed()) { + throw new ApiExecuteException(90101, "Failed to execute Job: " + jobInfo.getTask().get("errDesc").toString()); + } + } + + + public static void cancel(ApiServiceExecuteJob job,UJESClient client) { + client.kill(job.getJobExecuteResult()); + } + + + public static double getProgress(ApiServiceExecuteJob job,UJESClient client) { + return client.progress(job.getJobExecuteResult()).getProgress(); + } + + + public static Boolean isCompleted(ApiServiceExecuteJob job,UJESClient client) { + return client.getJobInfo(job.getJobExecuteResult()).isCompleted(); + } + + public static String getResult(ApiServiceExecuteJob job, int index, int maxSize, UJESClient client) { + String resultContent = null; + JobInfoResult jobInfo = client.getJobInfo(job.getJobExecuteResult()); + String[] resultSetList = jobInfo.getResultSetList(client); + if (resultSetList != null && resultSetList.length > 0) { + Object fileContent = client.resultSet(ResultSetAction.builder() + .setPath(resultSetList[index]) + .setUser(job.getJobExecuteResult().getUser()) + .setPageSize(maxSize).build()).getFileContent(); + if (fileContent instanceof ArrayList) { + ArrayList> resultSetRow = (ArrayList>) fileContent; + resultContent = StringUtils.join(resultSetRow.get(0), "\n"); + } else { + resultContent = fileContent.toString(); + } + } + return resultContent; + } + + + public static int getResultSize(ApiServiceExecuteJob job, UJESClient client) { + JobInfoResult jobInfo = client.getJobInfo(job.getJobExecuteResult()); + if (jobInfo.isSucceed()) { + String[] resultSetList = jobInfo.getResultSetList(client); + if (resultSetList != null && resultSetList.length > 0) { + return resultSetList.length; + } + } + return 0; + } + + + public static String getLog(ApiServiceExecuteJob job, UJESClient client) { + + JobLogResult jobLogResult = client + .log(job.getJobExecuteResult(), + 0, + 50); + + ArrayList logArray = jobLogResult.getLog(); + + if (logArray != null && logArray.size() + >= ApiServiceConfiguration.LOG_ARRAY_LEN.getValue() + && StringUtils.isNotEmpty(logArray.get(3))) { + return logArray.get(3); + } + return null; + } + + public static String getResultList(JobExecuteResult executeResult,UJESClient client, String path) { + ResultSetListResult resultList = (ResultSetListResult) client.executeUJESJob(ResultSetListAction.builder() + .setUser(executeResult.getUser()).setPath(path).build()); + return resultList.getResponseBody(); + } + + + + public static String getResultContent(String user, String path, int maxSize, UJESClient client) { + return client.resultSet(ResultSetAction.builder() + .setPath(path) + .setUser(user) + .setPageSize(maxSize).build()).getResponseBody(); + } + + public static InputStream downloadResultSet(String user, + String path, + String charset, + String outputFileType, + String csvSeperator, + String outputFileName, + String sheetName, + String nullValue, + UJESClient client) { + + ResultSetDownloadAction resultSetDownloadAction = new ResultSetDownloadAction(); + resultSetDownloadAction.setUser(user); + resultSetDownloadAction.setParameter("path",path); + resultSetDownloadAction.setParameter("charset",charset); + resultSetDownloadAction.setParameter("outputFileType",outputFileType); + resultSetDownloadAction.setParameter("csvSeperator",csvSeperator); + resultSetDownloadAction.setParameter("outputFileName",outputFileName); + resultSetDownloadAction.setParameter("sheetName",sheetName); + resultSetDownloadAction.setParameter("nullValue",nullValue); + client.executeUJESJob(resultSetDownloadAction); + return resultSetDownloadAction.getInputStream(); + } + + + + + + + + public static Map getTaskInfoById(JobExecuteResult jobExecuteResult, UJESClient client) { + return (Map) client.getJobInfo(jobExecuteResult).getTask(); + } + + + + + public static String getUserWorkspaceIds(String userName,UJESClient client){ + ApiServiceGetAction apiServiceGetAction = new ApiServiceGetAction(); + apiServiceGetAction.setUser(userName); + apiServiceGetAction.setParameter("userName",userName); + ResultWorkspaceIds userWorkspaceIds = (ResultWorkspaceIds)client.executeUJESJob(apiServiceGetAction); + return userWorkspaceIds.getUserWorkspaceIds(); + + } + + + + + + + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/LinkisJobSubmit.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/LinkisJobSubmit.java new file mode 100644 index 000000000..9c63d0247 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/LinkisJobSubmit.java @@ -0,0 +1,109 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.execute; + +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.UJESClientImpl; +import org.apache.linkis.ujes.client.request.JobExecuteAction; +import org.apache.linkis.ujes.client.response.JobExecuteResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + + +public class LinkisJobSubmit { + + private static final Map ujesClientMap = new HashMap<>(); + + public static UJESClient getClient() { + return getClient(new HashMap<>(0)); + } + + public static UJESClient getClient(Map params) { + return getUJESClient( + Configuration.GATEWAY_URL().getValue(params), + ApiServiceConfiguration.LINKIS_ADMIN_USER.getValue(params), + ApiServiceConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue(params), + params); + } + + + public static UJESClient getUJESClient(String url, String user, String token, Map jobProps){ + String key = url + user + token; + if(ujesClientMap.containsKey(key)) { + return ujesClientMap.get(key); + } + synchronized (LinkisJobSubmit.class) { + if(!ujesClientMap.containsKey(key)) { + ujesClientMap.put(key, new UJESClientImpl(getClientConfig(url,user,token, jobProps))); + } + } + return ujesClientMap.get(key); + } + + public static DWSClientConfig getClientConfig(String url, String user, String token, Map jobProps){ + return ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() + .addServerUrl(url) + .connectionTimeout(ApiServiceConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) + .discoveryEnabled(false).discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(5) + .retryEnabled(false).readTimeout(ApiServiceConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) + .setAuthTokenKey(user).setAuthTokenValue(token))) + .setDWSVersion(ApiServiceConfiguration.LINKIS_API_VERSION.getValue(jobProps)).build(); + } + + + + public static JobExecuteResult execute(ApiServiceExecuteJob job,UJESClient client) { + Map source = new HashMap<>(); + source.put("DSS-ApiService",job.getScriptPath()); + JobExecuteAction.Builder builder = JobExecuteAction.builder().setCreator("IDE") + .addExecuteCode(job.getCode()) + .setEngineTypeStr(job.getEngineType()) + .setRunTypeStr(job.getRunType()) + .setUser(job.getUser()) + .setParams(job.getParams()) + .setVariableMap(job.getRuntimeParams()) + .setSource(source); + JobExecuteAction jobAction = builder.build(); + JobExecuteResult res = client.execute(jobAction); + return res; + } + public static JobExecuteResult execute(ApiServiceExecuteJob job,UJESClient client, String creator) { + Map source = new HashMap<>(); + source.put("DSS-ApiService",job.getScriptPath()); + JobExecuteAction.Builder builder = JobExecuteAction.builder().setCreator(creator) + .addExecuteCode(job.getCode()) + .setEngineTypeStr(job.getEngineType()) + .setRunTypeStr(job.getRunType()) + .setUser(job.getUser()) + .setParams(job.getParams()) + .setVariableMap(job.getRuntimeParams()) + .setSource(source); + JobExecuteAction jobAction = builder.build(); + JobExecuteResult res = client.execute(jobAction); + return res; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/DatasourceService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/DatasourceService.java new file mode 100644 index 000000000..0b246ae86 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/DatasourceService.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.jdbc; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import javax.sql.DataSource; +import java.util.HashMap; +import java.util.Map; + + +@Configuration +public class DatasourceService { + private static final Map CLIENT_MAP = new HashMap<>(); + + + private DataSource getDatasource(String url, String username, String password) { + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(url); + hikariConfig.setUsername(username); + hikariConfig.setPassword(password); + hikariConfig.setMaximumPoolSize(100); + hikariConfig.setMinimumIdle(10); + return new HikariDataSource(hikariConfig); + } + + + public NamedParameterJdbcTemplate getNamedParameterJdbcTemplate(String url, + String username, String password) { + String key = genKey(url, username); + NamedParameterJdbcTemplate jdbcTemplate = CLIENT_MAP.get(key); + + if (jdbcTemplate == null) { + synchronized (this) { + jdbcTemplate = CLIENT_MAP.get(key); + if (jdbcTemplate == null) { + jdbcTemplate = new NamedParameterJdbcTemplate(getDatasource(url, username, password)); + CLIENT_MAP.put(key, jdbcTemplate); + } + } + } + + return jdbcTemplate; + } + + private String genKey(String jdbcUrl, String username) { + return String.join("-", jdbcUrl, username); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/JdbcUtil.scala b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/JdbcUtil.scala new file mode 100644 index 000000000..b897118e2 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/JdbcUtil.scala @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.jdbc + +import java.util + +import org.apache.linkis.common.exception.WarnException +import org.apache.linkis.common.utils.Logging +//import org.apache.linkis.datasourcemanager.common.protocol.{DsInfoQueryRequest, DsInfoResponse} +import org.apache.linkis.rpc.Sender + +object JdbcUtil extends Logging { + +// val sender : Sender = Sender.getSender("dsm-server") +// def getDatasourceInfo(params : util.Map[String, Any]) : (String, String, String) = { +// val datasourceId = params.get("configuration").asInstanceOf[util.Map[String, Any]] +// .getOrDefault("datasource", new util.HashMap[String, Any]()) +// .asInstanceOf[util.Map[String, Any]].get("datasourceId") +// logger.info(s"begin to get datasource info from dsm, datasourceId: ${datasourceId}") +// if (datasourceId != null) { +// val ds = sender.ask(DsInfoQueryRequest(String.valueOf(datasourceId), "BDP")) match { +// case r: DsInfoResponse => r +// case warn: WarnException => throw warn +// } +// logger.info(s"get datasource info result: ${ds}") +// if (ds.status) { +// val url = ds.params.get("jdbc.url").asInstanceOf[String] +// val userName = ds.params.get("jdbc.username").asInstanceOf[String] +// val password = ds.params.get("jdbc.password").asInstanceOf[String] +// logger.info(s"get from dsm: url: ${url}, username: ${userName}, password: ${password}") +// return (url, userName, password) +// } +// } +// +// ("", "", "") +// } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceCoreRestfulApi.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceCoreRestfulApi.java new file mode 100644 index 000000000..9fd4e2130 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceCoreRestfulApi.java @@ -0,0 +1,520 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.restful; + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiCommentUpdateRequest; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceQuery; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiService; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiServiceQueryService; +import com.webank.wedatasphere.dss.apiservice.core.util.ApiUtils; +import com.webank.wedatasphere.dss.apiservice.core.util.AssertUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiServiceVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiVersionVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.QueryParamVo; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import javax.validation.ConstraintViolation; +import javax.validation.ConstraintViolationException; +import javax.validation.Validator; +import javax.validation.groups.Default; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@RequestMapping(path = "/dss/apiservice", produces = {"application/json"}) +@RestController +public class ApiServiceCoreRestfulApi { + + private static final Logger LOG = LoggerFactory.getLogger(ApiServiceCoreRestfulApi.class); + + @Autowired + private ApiService apiService; + + @Autowired + private ApiServiceQueryService apiServiceQueryService; + + @Autowired + private Validator beanValidator; + + private static final Pattern WRITABLE_PATTERN = Pattern.compile("^\\s*(insert|update|delete|drop|alter|create).*", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + @RequestMapping(value = "/api",method = RequestMethod.POST) + public Message insert(@RequestBody ApiServiceVo apiService, HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + + if (apiService.getWorkspaceId() == null){ + apiService.setWorkspaceId(180L); + } + + if (StringUtils.isBlank(apiService.getAliasName())) { + return Message.error("'api service alias name' is missing[缺少中文名]"); + } + + if (StringUtils.isBlank(apiService.getScriptPath())) { + return Message.error("'api service script path' is missing[缺少脚本路径]"); + } + if (StringUtils.isBlank(apiService.getContent())) { + return Message.error("'api service script content' is missing[缺少脚本内容]"); + } + + if (null == apiService.getWorkspaceId()) { + return Message.error("'api service workspaceId ' is missing[缺少工作空间ID]"); + } + if (apiService.getContent().contains(";")) { + if(!apiService.getContent().toLowerCase().startsWith("use ")) { + return Message.error("'api service script content exists semicolon[脚本内容包含分号]"); + } + } + +// check data change script + if (WRITABLE_PATTERN.matcher(apiService.getContent()).matches()) { + return Message.error("'api service script content' only supports query[脚本内容只支持查询语句]"); + } + + Map metadata = apiService.getMetadata(); + if (apiService.getScriptPath().endsWith(".jdbc")) { + if (MapUtils.isEmpty(metadata)) { + return Message.error("'api service metadata' is missing[请选择数据源]"); + } + + Map configuration = (Map) metadata.get("configuration"); + if (MapUtils.isEmpty(configuration)) { + return Message.error("'api service metadata.configuration' is missing[请选择数据源]"); + } + + Map datasource = (Map) configuration.get("datasource"); + if (MapUtils.isEmpty(datasource)) { + return Message.error("'api service metadata.configuration.datasource' is missing[请选择数据源]"); + } + } + + String userName = SecurityFilter.getLoginUsername(req); + Set> result = beanValidator.validate(apiService, Default.class); + if (result.size() > 0) { + throw new ConstraintViolationException(result); + } + + ApprovalVo approvalVo = apiService.getApprovalVo(); + +// if (StringUtils.isBlank(approvalVo.getApprovalName())) { +// return Message.error("'approvalName' is missing[缺少审批单名字]"); +// } + + if (StringUtils.isBlank(approvalVo.getApplyUser())) { + return Message.error("'applyUser' is missing[缺少申请用户名字]"); + } + + apiService.setCreator(userName); + apiService.setModifier(userName); + this.apiService.save(apiService); + return Message.ok().data("insert_id", apiService.getId()).data("approval_no",approvalVo.getApprovalNo()); + }, "/apiservice/api", "Fail to insert service api[新增服务api失败]"); + } + + @RequestMapping(value = "/create",method = RequestMethod.POST) + public Message create(@RequestBody ApiServiceVo apiService, HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + + if (apiService.getWorkspaceId() == null){ + apiService.setWorkspaceId(180L); + } + + if (StringUtils.isBlank(apiService.getAliasName())) { + return Message.error("'api service alias name' is missing[缺少中文名]"); + } + + if (StringUtils.isBlank(apiService.getScriptPath())) { + return Message.error("'api service script path' is missing[缺少脚本路径]"); + } + if (StringUtils.isBlank(apiService.getContent())) { + return Message.error("'api service script content' is missing[缺少脚本内容]"); + } + + if (null == apiService.getWorkspaceId()) { + return Message.error("'api service workspaceId ' is missing[缺少工作空间ID]"); + } + if (apiService.getContent().contains(";")) { + if(!apiService.getContent().toLowerCase().startsWith("use ")) { + return Message.error("'api service script content exists semicolon[脚本内容包含分号]"); + } + } + +// check data change script + if (WRITABLE_PATTERN.matcher(apiService.getContent()).matches()) { + return Message.error("'api service script content' only supports query[脚本内容只支持查询语句]"); + } + + Map metadata = apiService.getMetadata(); + if (apiService.getScriptPath().endsWith(".jdbc")) { + if (MapUtils.isEmpty(metadata)) { + return Message.error("'api service metadata' is missing[请选择数据源]"); + } + + Map configuration = (Map) metadata.get("configuration"); + if (MapUtils.isEmpty(configuration)) { + return Message.error("'api service metadata.configuration' is missing[请选择数据源]"); + } + + Map datasource = (Map) configuration.get("datasource"); + if (MapUtils.isEmpty(datasource)) { + return Message.error("'api service metadata.configuration.datasource' is missing[请选择数据源]"); + } + } + + String userName = SecurityFilter.getLoginUsername(req); + Set> result = beanValidator.validate(apiService, Default.class); + if (result.size() > 0) { + throw new ConstraintViolationException(result); + } + + ApprovalVo approvalVo = apiService.getApprovalVo(); + + if (StringUtils.isBlank(approvalVo.getApprovalName())) { + return Message.error("'approvalName' is missing[缺少审批单名字]"); + } + + if (StringUtils.isBlank(approvalVo.getApplyUser())) { + return Message.error("'applyUser' is missing[缺少申请用户名字]"); + } + + apiService.setCreator(userName); + apiService.setModifier(userName); + this.apiService.saveByApp(apiService); + return Message.ok().data("insert_id", apiService.getId()).data("approval_no",approvalVo.getApprovalNo()); + }, "/apiservice/api", "Fail to insert service api[新增服务api失败]"); + } + + @RequestMapping(value = "/api/{api_service_version_id}",method = RequestMethod.PUT) + public Message update(@RequestBody ApiServiceVo apiService, + @PathVariable("api_service_version_id") Long apiServiceVersionId, + HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + + if (StringUtils.isBlank(apiService.getScriptPath())) { + return Message.error("'api service script path' is missing[缺少脚本路径]"); + } + if(apiServiceVersionId !=0) { + if (StringUtils.isBlank(apiService.getPath())) { + return Message.error("'api service api path' is missing[缺少api路径]"); + } + } + if (StringUtils.isBlank(apiService.getContent())) { + return Message.error("'api service script content' is missing[缺少脚本内容]"); + } + + if (null == apiService.getWorkspaceId()) { + return Message.error("'api service workspaceId ' is missing[缺少工作空间ID]"); + } + + if (null == apiService.getTargetServiceId()) { + return Message.error("'api service update to target service id ' is missing[缺少更新目标服务ID]"); + } + + if (apiService.getContent().contains(";")) { + return Message.error("'api service script content exists semicolon[脚本内容包含分号]"); + } + + ApprovalVo approvalVo = apiService.getApprovalVo(); + +// if (StringUtils.isBlank(approvalVo.getApprovalName())) { +// return Message.error("'approvalName' is missing[缺少审批单名字]"); +// } + + if (StringUtils.isBlank(approvalVo.getApplyUser())) { + return Message.error("'applyUser' is missing[缺少申请用户名字]"); + } +// if (StringUtils.isBlank(apiService.getResourceId())) { +// return Message.error("'api service resourceId' is missing[缺少bml resourceId]"); +// } + +// check data change script + if (WRITABLE_PATTERN.matcher(apiService.getContent()).matches()) { + return Message.error("'api service script content' only supports query[脚本内容只支持查询语句]"); + } + + Map metadata = apiService.getMetadata(); + if (apiService.getScriptPath().endsWith(".jdbc")) { + if (MapUtils.isEmpty(metadata)) { + return Message.error("'api service metadata' is missing[请选择数据源]"); + } + + Map configuration = (Map) metadata.get("configuration"); + if (MapUtils.isEmpty(configuration)) { + return Message.error("'api service metadata.configuration' is missing[请选择数据源]"); + } + + Map datasource = (Map) configuration.get("datasource"); + if (MapUtils.isEmpty(datasource)) { + return Message.error("'api service metadata.configuration.datasource' is missing[请选择数据源]"); + } + } + + String userName = SecurityFilter.getLoginUsername(req); +// Bean validation + Set> result = beanValidator.validate(apiService, Default.class); + if (result.size() > 0) { + throw new ConstraintViolationException(result); + } + apiService.setLatestVersionId(apiServiceVersionId); + apiService.setModifier(userName); + apiService.setModifyTime(Calendar.getInstance().getTime()); + this.apiService.update(apiService); + return Message.ok().data("update_id", apiServiceVersionId); + }, "/apiservice/api/" + apiServiceVersionId, "Fail to update service api[更新服务api失败]"); + } + + + + + @RequestMapping(value = "/search",method = RequestMethod.GET) + public Message query(@RequestParam(required = false, name = "name") String name, + @RequestParam(required = false, name = "tag") String tag, + @RequestParam(required = false, name = "status") Integer status, + @RequestParam(required = false, name = "creator") String creator, + @RequestParam(required = false, name = "workspaceId") Integer workspaceId, + HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + + return ApiUtils.doAndResponse(() -> { + if (null == workspaceId) { + return Message.error("'api service search workspaceId' is missing[缺少工作空间Id]"); + } + ApiServiceQuery query = new ApiServiceQuery(userName,name, tag, status, creator); + query.setWorkspaceId(workspaceId); + if(!this.apiService.checkUserWorkspace(userName,workspaceId) ){ + return Message.error("'api service search workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + List queryList = apiService.query(query); + return Message.ok().data("query_list", queryList); + }, "/apiservice/search", "Fail to query page of service api[查询服务api失败]"); + } + + + @RequestMapping(value = "/getUserServices",method = RequestMethod.GET) + public Message getUserServices(@RequestParam(required = false, name = "workspaceId") Integer workspaceId, + HttpServletRequest req){ + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + if(!this.apiService.checkUserWorkspace(userName,workspaceId) ){ + return Message.error("'api service getUserServices workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + List apiServiceList = apiService.queryByWorkspaceId(workspaceId,userName); + return Message.ok().data("query_list", apiServiceList); + }, "/apiservice/getUserServices", "Fail to query page of user service api[查询用户服务api列表失败]"); + } + + + + @RequestMapping(value = "/tags",method = RequestMethod.GET) + public Message query( HttpServletRequest req,@RequestParam(required = false, name = "workspaceId") Integer workspaceId) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + + List tags= apiService.queryAllTags(userName,workspaceId); + return Message.ok().data("tags", tags); + }, "/apiservice/tags", "Fail to query page of service tag[查询服务tag失败]"); + } + + + + + @RequestMapping(value = "/query",method = RequestMethod.GET) + public Message queryByScriptPath(@RequestParam(required = false, name = "scriptPath") String scriptPath, + HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isBlank(scriptPath)) { + return Message.error("'api service scriptPath' is missing[缺少脚本路径]"); + } + ApiServiceVo apiServiceVo = apiService.queryByScriptPath(scriptPath); + if(null != apiServiceVo) { + if (!this.apiService.checkUserWorkspace(userName, apiServiceVo.getWorkspaceId().intValue())) { + return Message.error("'api service query workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + + if (apiServiceVo.getCreator().equals(userName)) { + return Message.ok().data("result", apiServiceVo); + } else { + return Message.error("'api service belong to others' [该脚本路径数据服务不属于当前用户]"); + } + }else { + return Message.ok().data("result", apiServiceVo); + } + }, "/apiservice/query", "Fail to query page of service api[查询服务api失败]"); + } + + @RequestMapping(value = "/queryById",method = RequestMethod.GET) + public Message queryById(@RequestParam(required = false, name = "id") Long id, + HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + if (id==null) { + return Message.error("'api service id' is missing[缺少服务ID]"); + } + ApiServiceVo apiServiceVo = apiService.queryById(id,userName); + AssertUtil.notNull(apiServiceVo,"未找到数据服务,有可能已经被删除"); + if(!this.apiService.checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue()) ){ + return Message.error("'api service queryById for workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + return Message.ok().data("result", apiServiceVo); + }, "/apiservice/queryById", "Fail to query page of service api[查询服务api失败]"); + } + + @RequestMapping(value = "/checkPath",method = RequestMethod.GET) + public Message checkPath(@RequestParam(required = false, name = "scriptPath") String scriptPath, @RequestParam(required = false, name = "path") String path) { + //需要跨用户查询 + return ApiUtils.doAndResponse(() -> { + if (StringUtils.isBlank(scriptPath)) { + return Message.error("'api service scriptPath' is missing[缺少api脚本路径]"); + } + if (StringUtils.isBlank(path)) { + return Message.error("'api service path' is missing[缺少api路径]"); + } + Integer apiCount = apiService.queryCountByPath(scriptPath, path); + return Message.ok().data("result", 0 > Integer.valueOf(0).compareTo(apiCount)); + }, "/apiservice/checkPath", "Fail to check path of service api[校验服务api路径失败]"); + } + + @RequestMapping(value = "/checkName",method = RequestMethod.GET) + public Message checkName(@RequestParam(required = false, name = "name") String name) { + //需要跨用户查询 + return ApiUtils.doAndResponse(() -> { + if (StringUtils.isBlank(name)) { + return Message.error("'api service name' is missing[缺少api名称]"); + } + Integer count = apiService.queryCountByName(name); + return Message.ok().data("result", count > 0); + }, "/apiservice/checkName", "Fail to check name of service api[校验服务api名称失败]"); + } + + @RequestMapping(value = "/apiDisable",method = RequestMethod.GET) + public Message apiDisable(@RequestParam(required = false, name = "id") Long id, + HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == id) { + return Message.error("'api service api id' is missing[缺少api id]"); + } + boolean resultFlag = apiService.disableApi(id,userName); + return Message.ok().data("result", resultFlag); + }, "/apiservice/apiDisable", "Fail to disable api[禁用api失败]"); + } + + @RequestMapping(value = "/apiEnable",method = RequestMethod.GET) + public Message apiEnable(@RequestParam(required = false, name = "id") Long id, + HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == id) { + return Message.error("'api service api id' is missing[缺少api id]"); + } + boolean resultFlag = apiService.enableApi(id,userName); + return Message.ok().data("result", resultFlag); + }, "/apiservice/apiEnable", "Fail to enable api[启用api失败]"); + } + + @RequestMapping(value = "/apiDelete",method = RequestMethod.GET) + public Message apiDelete(@RequestParam(required = false, name = "id") Long id, + HttpServletRequest req) { + //目前暂时不实际删除数据,只做不可见和不可用。 + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == id) { + return Message.error("'api service api id' is missing[缺少api id]"); + } + boolean resultFlag = apiService.deleteApi(id,userName); + return Message.ok().data("result", resultFlag); + }, "/apiservice/apiDelete", "Fail to delete api[删除api失败]"); + } + + @RequestMapping(value = "/apiCommentUpdate",method = RequestMethod.POST) + public Message apiCommentUpdate(HttpServletRequest req, + @RequestBody ApiCommentUpdateRequest apiCommentUpdateRequest) { + Long id = apiCommentUpdateRequest.getId(); + String comment = apiCommentUpdateRequest.getComment(); + //目前暂时不实际删除数据,只做不可见和不可用。 + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == id) { + return Message.error("'api service api id' is missing[缺少api id]"); + } + boolean resultFlag = apiService.updateComment(id,comment,userName); + return Message.ok().data("result", resultFlag); + }, "/apiservice/apiDelete", "Fail to delete api[删除api失败]"); + } + + + @RequestMapping(value = "/apiParamQuery",method = RequestMethod.GET) + public Message apiParamQuery(@RequestParam(required = false, name = "scriptPath") String scriptPath, + @RequestParam(required = false, name = "versionId") Long versionId, + HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(scriptPath)) { + return Message.error("'api service api scriptPath' is missing[缺少api scriptPath]"); + } + if (null == versionId) { + return Message.error("'api service api version' is missing[缺少api 版本号]"); + } + List queryParamVoList = apiServiceQueryService.queryParamList(scriptPath, versionId); + return Message.ok().data("result", queryParamVoList); + }, "/apiservice/apiParamQuery", "Fail to query api info[查询api信息失败]"); + } + + @RequestMapping(value = "/apiVersionQuery",method = RequestMethod.GET) + public Message apiVersionQuery(@RequestParam(required = false, name = "serviceId") Long serviceId, + HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == serviceId) { + return Message.error("'api service api serviceId' is missing[缺少api serviceId]"); + } + List apiVersionVoList = apiServiceQueryService.queryApiVersionById(serviceId) + .stream().filter(apiVersionVo -> apiVersionVo.getCreator().equals(userName)) + .collect(Collectors.toList()); + return Message.ok().data("result", apiVersionVoList); + }, "/apiservice/apiVersionQuery", "Fail to query api version[查询api版本失败]"); + } + + @RequestMapping(value = "/apiContentQuery",method = RequestMethod.GET) + public Message apiContentQuery(@RequestParam(required = false, name = "versionId") Long versionId, + HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + if (null== versionId) { + return Message.error("'api service api versionId' is missing[缺少api versionId]"); + } + ApiServiceVo apiServiceVo = apiServiceQueryService.queryByVersionId(userName,versionId); + if(!this.apiService.checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue()) ){ + return Message.error("'api service apiContentQuery for workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + return Message.ok().data("result", apiServiceVo); + }, "/apiservice/apiContentQuery", "Fail to query api Content[查询api版本内容失败]"); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java new file mode 100644 index 000000000..91e427824 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java @@ -0,0 +1,356 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.restful; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.webank.wedatasphere.dss.apiservice.core.bo.*; +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.execute.ExecuteCodeHelper; +import com.webank.wedatasphere.dss.apiservice.core.execute.LinkisJobSubmit; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiServiceQueryService; +import com.webank.wedatasphere.dss.apiservice.core.token.JwtManager; +import com.webank.wedatasphere.dss.apiservice.core.util.ApiUtils; +import com.webank.wedatasphere.dss.apiservice.core.util.AssertUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.MessageVo; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.io.InputStream; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +@RequestMapping(path = "/dss/apiservice", produces = {"application/json"}) +@RestController +public class ApiServiceExecuteRestfulApi { + public static final String XLSX_RESPONSE_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + + private static final Logger logger = LoggerFactory.getLogger(ApiServiceExecuteRestfulApi.class); + @Autowired + private ApiServiceQueryService queryService; + private static final String SYS_COLUMN_PREFIX = "_"; + + private static final String requestBodyDemo = "{\"moduleName\":\"aladdin-demo\",\"params\":{\"param1\": \"value1\"}}"; + + @RequestMapping(value = "/execute/{path:.*}",method = RequestMethod.POST) + public Message post(@PathVariable("path") VariableString path, @RequestBody QueryRequest queryRequest, + HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return getResponse(userName,path.getPath(), queryRequest, HttpMethod.POST); + } + + @RequestMapping(value = "/execute/{path:.*}",method = RequestMethod.GET) + public Message get(@PathVariable("path") VariableString path, + HttpServletRequest req) throws JsonProcessingException { + String userName = SecurityFilter.getLoginUsername(req); + + QueryRequest queryRequest = new QueryRequest(); + + + // 如果requestBody为空,尝试从url中获取参数 + if (MapUtils.isEmpty(queryRequest.getParams())) { + String paramSuffix = "params."; + Enumeration parameterNames = req.getParameterNames(); + Map params = new HashMap<>(); + while (parameterNames.hasMoreElements()) { + String name = parameterNames.nextElement(); + if (name.startsWith(paramSuffix)) { + params.put(name.substring(paramSuffix.length()), req.getParameter(name)); + } + } + queryRequest.setParams(params); + } + String paramsJsonStr = req.getParameter("params"); + JavaType javaType = BDPJettyServerHelper.jacksonJson().getTypeFactory().constructParametricType(Map.class,String.class,Object.class); + Map parmas = BDPJettyServerHelper.jacksonJson().readValue(paramsJsonStr, javaType); + queryRequest.setParams(parmas); + + if (StringUtils.isEmpty(queryRequest.getModuleName())) { + queryRequest.setModuleName(req.getParameter("moduleName")); + } + + return getResponse(userName,path.getPath(), queryRequest, HttpMethod.GET); + } + + @RequestMapping(value = "/execute/{path:.*}",method = RequestMethod.PUT) + public Message put(@PathVariable("path") VariableString path, @RequestBody QueryRequest queryRequest, + HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return getResponse(userName,path.getPath(), queryRequest, HttpMethod.PUT); + } + + @RequestMapping(value = "/execute/{path:.*}",method = RequestMethod.DELETE) + public Message delete(@PathVariable("path") VariableString path, @RequestBody QueryRequest queryRequest, + HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return getResponse(userName,path.getPath(), queryRequest, HttpMethod.DELETE); + } + + private void validParam(QueryRequest queryRequest) { + AssertUtil.notNull(queryRequest, "请求体不能为空,正确的格式:" + requestBodyDemo); + AssertUtil.notEmpty(queryRequest.getModuleName(), "moduleName不能为空,正确的格式:" + requestBodyDemo); + AssertUtil.notNull(queryRequest.getParams().get(ApiServiceConfiguration.API_SERVICE_TOKEN_KEY.getValue()),"请求token不能为空"); + } + + @RequestMapping(value = "/getDirFileTrees",method = RequestMethod.GET) + public Message getDirFileTrees(HttpServletRequest req, HttpServletResponse resp, + @RequestParam(required = false, name = "path") String path, + @RequestParam(required = false, name = "taskId") String taskId) { + String userName = SecurityFilter.getLoginUsername(req); + if (!isNumber(taskId)) { + return Message.error("请求参数 taskId 非法."); + } else if(StringUtils.isEmpty(taskId)){ + return Message.error("taskId 为空."); + } + if (StringUtils.isEmpty(path)) { + return Message.error("path 为空."); + } + ApiServiceJob apiServiceJob = queryService.getJobByTaskId(taskId); + if(apiServiceJob == null) { + return Message.error("当前用户不存在运行的TaskId: " + taskId); + } else if(userName.equals(apiServiceJob.getSubmitUser())) { + JobExecuteResult jobExecuteResult = new JobExecuteResult(); + jobExecuteResult.setTaskID(taskId); + jobExecuteResult.setUser(apiServiceJob.getProxyUser()); + UJESClient client = LinkisJobSubmit.getClient(); + String dirFileTree = ExecuteCodeHelper.getResultList(jobExecuteResult, client, path); + return DSSCommonUtils.COMMON_GSON.fromJson(dirFileTree, Message.class); + } else { + return Message.error("You are not the submitUser, cannot open the resultSet."); + } + } + + private void writeMessage(HttpServletResponse resp, Message message) throws IOException { + if(message == null) { + return; + } else { + resp.setStatus(Message.messageToHttpStatus(message)); + } + String str = DSSCommonUtils.COMMON_GSON.toJson(message); + resp.getWriter().println(str); + resp.getWriter().flush(); + } + + @RequestMapping(value = "/openFile",method = RequestMethod.GET) + public Message openFile(HttpServletRequest req, + @RequestParam(required = false, name = "path") String path, + @RequestParam(required = false, name = "taskId") String taskId, + @RequestParam(required = false, name = "page", defaultValue = "1") Integer page, + @RequestParam(required = false, name = "pageSize", defaultValue = "5000") Integer pageSize, + @RequestParam(required = false, name = "charset", defaultValue = "utf-8") String charset) { + String userName = SecurityFilter.getLoginUsername(req); + logger.info("User {} wants to open resultSet file {} in task {}.", userName, path, taskId); + if (!isNumber(taskId)) { + return Message.error("请求参数 taskId 非法."); + } else if(StringUtils.isEmpty(taskId)){ + return Message.error("taskId 为空."); + } + if (StringUtils.isEmpty(path)) { + return Message.error("path 为空."); + } + + ApiServiceJob apiServiceJob = queryService.getJobByTaskId(taskId); + if(apiServiceJob == null) { + return Message.error("您不存在运行的TaskId: "+taskId); + } else if (userName.equals(apiServiceJob.getSubmitUser())) { + UJESClient client = LinkisJobSubmit.getClient(); + try { + String fileContent = ExecuteCodeHelper.getResultContent(apiServiceJob.getProxyUser(), path, pageSize, client); + return DSSCommonUtils.COMMON_GSON.fromJson(fileContent, Message.class); + } catch (Exception e) { + logger.error("User {} fetch resultSet {} failed.", userName, path, e); + return Message.error("Get resultSet failed! Reason: " + ExceptionUtils.getRootCauseMessage(e)); + } + } else { + return Message.error("You are not the submitUser, cannot open the resultSet."); + } + } + + @RequestMapping(value = "resultsetToExcel",method = RequestMethod.GET) + public void resultsetToExcel( + HttpServletRequest req, + HttpServletResponse resp, + @RequestParam(required = false, name = "path") String path, + @RequestParam(required = false, name = "taskId") String taskId, + @RequestParam(required = false, name = "charset", defaultValue = "utf-8") String charset, + @RequestParam(required = false, name = "outputFileType", defaultValue = "csv") String outputFileType, + @RequestParam(required = false, name = "csvSeperator", defaultValue = ",") String csvSeperator, + @RequestParam(required = false, name = "outputFileName", defaultValue = "downloadResultset") String outputFileName, + @RequestParam(required = false, name = "sheetName", defaultValue = "result") String sheetName, + @RequestParam(required = false, name = "nullValue", defaultValue = "NULL") String nullValue) throws ApiServiceQueryException, IOException { + String userName = SecurityFilter.getLoginUsername(req); + logger.info("User {} wants to download resultSet file {} as {} in task {}.", userName, path, outputFileType, taskId); + if (!isNumber(taskId)) { + writeMessage(resp, Message.error("请求参数 taskId 非法.")); + return; + } else if(StringUtils.isEmpty(taskId)){ + writeMessage(resp, Message.error("taskId 为空.")); + return; + } + if (StringUtils.isEmpty(path)) { + writeMessage(resp, Message.error("path 为空.")); + return; + } + resp.addHeader("Content-Disposition", "attachment;filename=" + + new String(outputFileName.getBytes("UTF-8"), "ISO8859-1") + "." + outputFileType); + resp.setCharacterEncoding(charset); + + switch (outputFileType) { + case "csv": + resp.addHeader("Content-Type", "text/plain"); + break; + case "xlsx": + resp.addHeader("Content-Type", XLSX_RESPONSE_CONTENT_TYPE); + break; + default: + writeMessage(resp, Message.error("不支持的下载类型.")); + return; + + } + + InputStream inputStream = null; + ApiServiceJob apiServiceJob = queryService.getJobByTaskId(taskId); + if(null == apiServiceJob) { + writeMessage(resp, Message.error("您不存在运行的TaskId: "+taskId)); + return; + } else if(userName.equals(apiServiceJob.getSubmitUser())) { + UJESClient client = LinkisJobSubmit.getClient(); + inputStream = ExecuteCodeHelper.downloadResultSet(apiServiceJob.getProxyUser(), + path, + charset, + outputFileType, + csvSeperator, + outputFileName, + sheetName, + nullValue, + client); + } + try { + IOUtils.copy(inputStream, resp.getOutputStream()); + resp.getOutputStream().flush(); + } finally { + IOUtils.closeQuietly(inputStream); + } + } + + @RequestMapping(value = "/{id}/get",method = RequestMethod.GET) + public Message getTaskByID(HttpServletRequest req, @PathVariable("id") Long taskId) { + if(taskId == null || !isNumber(taskId.toString())){ + return Message.error("请求参数taskId非法"); + } + String username = SecurityFilter.getLoginUsername(req); + ApiServiceJob apiServiceJob = queryService.getJobByTaskId(taskId.toString()); + if (null != apiServiceJob && username.equals(apiServiceJob.getSubmitUser())) { + UJESClient client = LinkisJobSubmit.getClient(); + JobExecuteResult jobExecuteResult = apiServiceJob.getJobExecuteResult(); + jobExecuteResult.setUser(apiServiceJob.getProxyUser()); + Map vo = ExecuteCodeHelper.getTaskInfoById(jobExecuteResult, client); + return Message.ok().data("task", vo); + } else { + return Message.ok().data("task", null); + } + } + + private Message getResponse(String user,String path, QueryRequest queryRequest, String httpMethod) { + Response response = ApiUtils.doAndResponse(() -> { + validParam(queryRequest); + String token = queryRequest.getParams().get(ApiServiceConfiguration.API_SERVICE_TOKEN_KEY.getValue()).toString(); + + MessageVo messageVo = null; + ApiServiceToken tokenDetail = null; + boolean isParseRight = true; + try { + tokenDetail = JwtManager.parseToken(token); + }catch (Exception e) { + isParseRight = false; + messageVo = new MessageVo().setData("token解析错误,该token无效!"); + } + if(false == isParseRight) { + return messageVo; + } + + if(tokenDetail.getApplyUser().equals(user)) { + LinkisExecuteResult query = queryService.query("/" + path, + queryRequest.getParams() == null ? new HashMap<>() : queryRequest.getParams(), + queryRequest.getModuleName(), httpMethod,tokenDetail,user); + if(null == query) { + messageVo = new MessageVo().setMessage("用户任务执行出错,用户参数错误!").setStatus(1); + return messageVo; + } + + HashMap queryRes = new HashMap<>(); + queryRes.put("taskId",query.getTaskId()); + queryRes.put("execId",query.getExecId()); + messageVo = new MessageVo().setData(queryRes); + }else { + messageVo = new MessageVo().setData("Token is not correct"); + } + return messageVo; + }); + return convertMessage(response); + } + + //convert Response to Message + public Message convertMessage(Response response) { + MessageVo tempVo = (MessageVo) response.getEntity(); + Message message = null; + if (tempVo.getStatus().intValue() == 1) { + message = Message.error(tempVo.getMessage()); + } else { + message = Message.ok(tempVo.getMessage()); + } + HashMap queryRes = (HashMap) tempVo.getData(); + if (!CollectionUtils.isEmpty(queryRes)) { + for (String key : queryRes.keySet()) { + message.data(key, queryRes.get(key)); + } + } + return message; + } + + Pattern numberPattern = Pattern.compile("^\\d+$"); + //Judge if the taskId is number + public boolean isNumber(String taskId) { + if (taskId == null || taskId.trim().equals("")) { + return false; + } + boolean matches = numberPattern.matcher(taskId).matches(); + return matches; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceTokenRestfulApi.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceTokenRestfulApi.java new file mode 100644 index 000000000..e0dbaa3b4 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceTokenRestfulApi.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.restful; + +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.apiservice.core.bo.TokenQuery; +import com.webank.wedatasphere.dss.apiservice.core.service.TokenQueryService; +import com.webank.wedatasphere.dss.apiservice.core.token.TokenAuth; +import com.webank.wedatasphere.dss.apiservice.core.util.ApiUtils; +import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import java.util.Calendar; +import java.util.Date; + + + +@RequestMapping(path = "/dss/apiservice", produces = {"application/json"}) +@RestController +public class ApiServiceTokenRestfulApi { + private static final Logger LOG = LoggerFactory.getLogger(ApiServiceTokenRestfulApi.class); + @Autowired + TokenQueryService tokenQueryService; + @Autowired + TokenAuth tokenAuth; + + @RequestMapping(value = "/tokenQuery",method = RequestMethod.GET) + public Message apiServiceTokenQuery(@RequestParam(required = false, name = "apiId") Long apiId, + @RequestParam(required = false, name = "user") String user, + @RequestParam(required = false, name = "status") Integer status, + @RequestParam(required = false, name = "startDate") String startDateStr, + @RequestParam(required = false, name = "endDate") String endDateStr, + @RequestParam(required = false, name = "currentPage") Integer currentPage, + @RequestParam(required = false, name = "pageSize") Integer pageSize, + HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + + TokenQuery query = null; + + if(StringUtils.isBlank(startDateStr) || StringUtils.isBlank(endDateStr)) { + query = new TokenQuery(apiId, user, status); + } else { + Long startDateSecond = Long.parseLong(startDateStr); + Long endDateSeconed = Long.parseLong(endDateStr); + String startDate = DateUtil.format(new Date(startDateSecond), DateUtil.FORMAT_LONG); + Date endDateReal = new Date(endDateSeconed); + Calendar cal = Calendar.getInstance(); + cal.setTime(endDateReal); + cal.add(Calendar.DATE, 1); + Date realDate = cal.getTime(); + String endDate = DateUtil.format(realDate, DateUtil.FORMAT_LONG); + + query = new TokenQuery(apiId, user, status, startDate, endDate); + + } + query.setCreator(userName); + query.setCurrentPage(null != currentPage ? currentPage : 1); + query.setPageSize(null != pageSize ? pageSize : 10); + PageInfo tokenList = tokenQueryService.query(query); + return Message.ok().data("queryList", tokenList.getList()).data("total", tokenList.getTotal()); + }, "/apiservice/tokenQuery", "Fail to query page of token[查询token信息失败]"); + } + + + @RequestMapping(value = "/approvalRefresh",method = RequestMethod.GET) + public Message refresh(@RequestParam(required = false, name = "approvalNo") String approvalNo, + HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() ->{ + return Message.ok().data("approvalStatus", "success"); + }, "/apiservice/approvalRefresh/", "查询出错"); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiService.java new file mode 100644 index 000000000..8937efaeb --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiService.java @@ -0,0 +1,98 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service; + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceQuery; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; + +import java.util.List; + + +public interface ApiService { + + /** + * Save + * + * @param oneService oneService info + */ + void save(ApiServiceVo oneService) throws Exception; + + + + void saveByApp(ApiServiceVo apiService) throws Exception; + + /** + * Update + * + * @param oneService oneService info + */ + void update(ApiServiceVo oneService) throws Exception; + + /** + * query + * + * @param apiServiceQuery + * @return + */ + List query(ApiServiceQuery apiServiceQuery) throws ApiServiceQueryException; + + List queryByWorkspaceId(Integer workspaceId, String userName); + + List queryAllTags(String userName,Integer workspaceId); + + + /** + * query + * + * @param scriptPath + * @return + */ + ApiServiceVo queryByScriptPath(String scriptPath); + + Integer queryCountByPath(String scriptPath, String path); + + Integer queryCountByName(String name); + + /** + * enable api + * @param id api record id + * @return + */ + Boolean enableApi(Long id,String userName); + + /** + * disable api + * @param id api record id + * @return + */ + Boolean disableApi(Long id,String userName); + + Boolean deleteApi(Long id,String userName); + + Boolean updateComment(Long id,String comment,String userName); + + ApiServiceVo queryById(Long id,String userName); + + ApiVersionVo getMaxVersion(long serviceId); + + boolean checkUserWorkspace(String userName,Integer workspaceId); + +// List genDataMapApplyContentDatas(ApiServiceVo apiServiceVo, ApiVersionVo apiVersionVo, String metaDtaInfo); + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiServiceQueryService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiServiceQueryService.java new file mode 100644 index 000000000..2f98390a9 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiServiceQueryService.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.service; + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceJob; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.bo.LinkisExecuteResult; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; + +import java.util.List; +import java.util.Map; + + +public interface ApiServiceQueryService { + /** + * 执行查询 + * + * @param path path + * @param params params + * @param moduleName moduleName + * @param httpMethod httpMethod + * @return 查询接口 + */ + LinkisExecuteResult query(String path, Map params, String moduleName, String httpMethod, ApiServiceToken tokenDetail, String loginUser); + + List queryParamList(String scriptPath, Long versionId); + + /** + * 查询api 版本信息 + * + * @param serviceId + * @return + */ + List queryApiVersionById(Long serviceId); + + ApiServiceVo queryByVersionId(String userName,Long versionId) throws ApiServiceQueryException; + + ApiServiceJob getJobByTaskId(String taskId); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApprovalService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApprovalService.java new file mode 100644 index 000000000..11ac1d192 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApprovalService.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service; + +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; + +public interface ApprovalService { + + /** + * 根据审批单号,查询对应数据 + * */ + ApprovalVo query(String approvalNo); + + /** + * 查询DataMap,依据DataMap审批结果更新审批单状态 + * */ + ApprovalVo refreshStatus(String approvalNo) throws Exception; +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/TokenQueryService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/TokenQueryService.java new file mode 100644 index 000000000..58e0a3014 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/TokenQueryService.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service; + +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import com.webank.wedatasphere.dss.apiservice.core.bo.TokenQuery; + + +public interface TokenQueryService { + + /** + * 查询 + * */ + PageInfo query(TokenQuery tokenQuery); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java new file mode 100644 index 000000000..426de81a2 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java @@ -0,0 +1,628 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.service.impl; + +import com.google.common.base.Splitter; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceQuery; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.constant.ApiCommonConstant; +import com.webank.wedatasphere.dss.apiservice.core.constant.SQLMetadataInfoCheckStatus; +import com.webank.wedatasphere.dss.apiservice.core.constant.SaveTokenEnum; +import com.webank.wedatasphere.dss.apiservice.core.dao.*; +import com.webank.wedatasphere.dss.apiservice.core.util.UUIDGenerator; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceTokenException; +import com.webank.wedatasphere.dss.apiservice.core.execute.LinkisJobSubmit; +import com.webank.wedatasphere.dss.apiservice.core.token.JwtManager; +import com.webank.wedatasphere.dss.apiservice.core.token.TokenAuth; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiService; +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.bml.protocol.BmlUpdateResponse; +import org.apache.linkis.bml.protocol.BmlUploadResponse; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.storage.script.ScriptFsWriter; +import org.apache.linkis.storage.script.ScriptMetaData; +import org.apache.linkis.storage.script.ScriptRecord; +import org.apache.linkis.storage.script.Variable; +import org.apache.linkis.storage.script.VariableParser; +import org.apache.linkis.storage.script.writer.StorageScriptFsWriter; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.commons.lang.StringUtils; +import org.apache.http.Consts; +import org.apache.ibatis.annotations.Param; +//import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.stream.Collectors; + + +@Service +public class ApiServiceImpl implements ApiService { + + private static final Logger LOG = LoggerFactory.getLogger(ApiServiceImpl.class); + + + @Autowired + private ApiServiceDao apiServiceDao; + + @Autowired + private ApiServiceParamDao apiServiceParamDao; + + @Autowired + private ApiServiceVersionDao apiServiceVersionDao; + + @Autowired + private ApiServiceTokenManagerDao apiServiceTokenManagerDao; + + @Autowired + TokenAuth tokenAuth; + + + + + /** + * Bml client + */ + private BmlClient client; + + private UJESClient ujesClient; + + @PostConstruct + public void buildClient() { + LOG.info("build client start ======"); + client = BmlClientFactory.createBmlClient(); + ujesClient = LinkisJobSubmit.getClient(); + LOG.info("build client end ======="); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void save(ApiServiceVo apiService) throws Exception { + String user = apiService.getCreator(); + String resourceId = null; + try { + // check script path if already created + String scriptPath = apiService.getScriptPath(); + + // upload to bml + Map uploadResult = uploadBml(user, scriptPath, + apiService.getMetadata(), apiService.getContent()); + + // insert linkis_oneservice_config + String version = uploadResult.get("version"); + resourceId = uploadResult.get("resourceId"); + apiServiceDao.insert(apiService); + + //insert into version + ApiVersionVo apiVersionVo = new ApiVersionVo(); + apiVersionVo.setApiId(apiService.getId()); + apiVersionVo.setBmlResourceId(resourceId); + apiVersionVo.setBmlVersion(version); + apiVersionVo.setVersion(version); + apiVersionVo.setCreator(user); + apiVersionVo.setCreateTime(Calendar.getInstance().getTime()); + apiVersionVo.setSource(apiService.getScriptPath()); + //1为正常 0为禁用 + apiVersionVo.setStatus(1); + + String approvalNo = UUIDGenerator.genUUID(); + apiVersionVo.setAuthId(approvalNo); + + //顺序不能改变,版本信息依赖审批单信息 + //todo update by query + apiVersionVo.setMetadataInfo("default"); + apiServiceVersionDao.insert(apiVersionVo); + + + + // insert linkis_oneservice_params + List params = apiService.getParams(); + if (params != null && !params.isEmpty()) { + for (ParamVo param : params) { + param.setApiVersionId(apiVersionVo.getId()); + apiServiceParamDao.insert(param); + } + } + + //insert a token record for self + genTokenForPublisher(apiService,apiVersionVo.getId()); + } catch (Exception e) { + LOG.error("one service insert error", e); + if (StringUtils.isNotBlank(resourceId)) { +// removeBml(user, resourceId); + } + if (e.getCause() instanceof ErrorException) { + throw (ErrorException) e.getCause(); + } + throw e; + } + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public void saveByApp(ApiServiceVo apiService) throws Exception { + String user = apiService.getCreator(); + String resourceId = null; + try { + // check script path if already created + String scriptPath = apiService.getScriptPath(); + + // upload to bml + Map uploadResult = uploadBml(user, scriptPath, + apiService.getMetadata(), apiService.getContent()); + + // insert linkis_oneservice_config + String version = uploadResult.get("version"); + resourceId = uploadResult.get("resourceId"); + apiServiceDao.insert(apiService); + + //insert into version + ApiVersionVo apiVersionVo = new ApiVersionVo(); + apiVersionVo.setApiId(apiService.getId()); + apiVersionVo.setBmlResourceId(resourceId); + apiVersionVo.setBmlVersion(version); + apiVersionVo.setVersion(version); + apiVersionVo.setCreator(user); + apiVersionVo.setCreateTime(Calendar.getInstance().getTime()); + apiVersionVo.setSource(apiService.getScriptPath()); + //1为正常 0为禁用 + apiVersionVo.setStatus(1); + + + //生成审批记录,必须使用发布用户执行sql +// checkApprovalFromDM(user,apiService,apiVersionVo); + //顺序不能改变,版本信息依赖审批单信息 + apiServiceVersionDao.insert(apiVersionVo); + +// addApprovalToDB(apiService,apiVersionVo.getId(),apiVersionVo.getAuthId()); + + // insert linkis_oneservice_params + List params = apiService.getParams(); + if (params != null && !params.isEmpty()) { + for (ParamVo param : params) { + param.setApiVersionId(apiVersionVo.getId()); + apiServiceParamDao.insert(param); + } + } + + //insert a token record for self + genTokenForPublisher(apiService,apiVersionVo.getId()); + } catch (Exception e) { + LOG.error("one service insert error", e); + if (StringUtils.isNotBlank(resourceId)) { +// removeBml(user, resourceId); + } + if (e.getCause() instanceof ErrorException) { + throw (ErrorException) e.getCause(); + } + throw e; + } + } + + + + + + @Override + @Transactional(rollbackFor = Exception.class) + public void update(ApiServiceVo apiService) throws Exception { + try { + if(null!=apiService.getTargetServiceId()) { + ApiVersionVo maxTargetApiVersionVo = getMaxVersion(apiService.getTargetServiceId()); + if(!checkUserWorkspace(apiService.getModifier(),apiService.getWorkspaceId().intValue())){ + throw new ApiServiceQueryException(800035,"Only can update the api service by owner workspace! "); + } + if(maxTargetApiVersionVo.getCreator().equals(apiService.getModifier())) { + Map updateResult = updateBml(apiService.getModifier(), maxTargetApiVersionVo.getBmlResourceId(), + apiService.getScriptPath(), apiService.getMetadata(), apiService.getContent()); + apiService.setCreator(maxTargetApiVersionVo.getCreator()); + apiService.setId(maxTargetApiVersionVo.getApiId()); + apiServiceDao.updateToTarget(apiService); +// Log.info("Update to other Api Service, ID: " + apiService.getTargetServiceId() + ",resourceId:" + maxTargetApiVersionVo.getBmlResourceId()); + + + String version = updateResult.get("version"); + String resourceId = updateResult.get("resourceId"); + + + //update api version + //insert into version + ApiVersionVo apiServiceVersionVo = new ApiVersionVo(); + apiServiceVersionVo.setApiId(apiService.getId()); + apiServiceVersionVo.setBmlResourceId(resourceId); + apiServiceVersionVo.setBmlVersion(version); + apiServiceVersionVo.setVersion(version); + apiServiceVersionVo.setCreator(apiService.getModifier()); + apiServiceVersionVo.setCreateTime(Calendar.getInstance().getTime()); + apiServiceVersionVo.setSource(apiService.getScriptPath()); + //0默认已禁用 1为正常 + apiServiceVersionVo.setStatus(1); + String approvalNo = UUIDGenerator.genUUID(); + apiServiceVersionVo.setAuthId(approvalNo); + + //todo update by query + apiServiceVersionVo.setMetadataInfo("null"); + //顺序不能改变,版本信息依赖审批单信息 + apiServiceVersionDao.insert(apiServiceVersionVo); + + //改变历史版本状态 + apiServiceVersionDao.updateApiVersionStatusById(maxTargetApiVersionVo.getId(), 0); + + //改变历史Token状态 + apiServiceTokenManagerDao.disableTokenStatusByVersionId(maxTargetApiVersionVo.getId()); + // insert params + List params = apiService.getParams(); + if (params != null && !params.isEmpty()) { + for (ParamVo param : params) { + param.setApiVersionId(apiServiceVersionVo.getId()); + apiServiceParamDao.insert(param); + } + } + +// addApprovalToDB(apiService, apiServiceVersionVo.getId(), apiServiceVersionVo.getAuthId()); + + //insert a token record for self + genTokenForPublisher(apiService, apiServiceVersionVo.getId()); + }else { + throw new ApiServiceQueryException(800036,"Only can update the api service by owner! "); + } + }else { + throw new ApiServiceQueryException(800037,"Target service id can not be null for update"); + } + + } catch (Exception e) { + LOG.error("api service update error", e); + if (e.getCause() instanceof ErrorException) { + throw (ErrorException) e.getCause(); + } + throw e; + } + } + + @Override + public List query(ApiServiceQuery apiServiceQuery) throws ApiServiceQueryException { + //todo 查询需要优化,量小时不影响效率 + List queryList = apiServiceDao.query(apiServiceQuery); + + //add auth check + List userTokenManagerVos = apiServiceTokenManagerDao.queryByApplyUser(apiServiceQuery.getUserName()); + + List authQueryList = queryList.stream().filter(apiServiceVo -> { + TokenManagerVo findUserTokenManagerVo = userTokenManagerVos.stream().filter(userTokenManagerVo -> + userTokenManagerVo.getApiId().equals(apiServiceVo.getId()) + && userTokenManagerVo.getUser().equals(apiServiceQuery.getUserName()) + ).findAny().orElse(null); + if (null != findUserTokenManagerVo) { + //检查token状态为有效的 + if(findUserTokenManagerVo.getStatus().equals(1) || findUserTokenManagerVo.getPublisher().equals(apiServiceVo.getCreator())) { + return true; + }else { + return false; + } + } else { + return false; + } + }).collect(Collectors.toList()); + + // query param + if (authQueryList != null && !authQueryList.isEmpty()) { + for (ApiServiceVo apiServiceVo : authQueryList) { + ApiVersionVo maxApiVersionVo = getMaxVersion(apiServiceVo.getId()); + if(null == maxApiVersionVo){ + throw new ApiServiceQueryException(800032,"数据服务API版本记录为空"+apiServiceVo.getName()); + } + apiServiceVo.setParams(apiServiceParamDao.queryByVersionId(maxApiVersionVo.getId())); + apiServiceVo.setLatestVersionId(maxApiVersionVo.getId()); + } + } + + return authQueryList; + } + + @Override + public List queryByWorkspaceId(Integer workspaceId, String userName){ + List result = apiServiceDao.queryByWorkspaceId(workspaceId,userName); + return result; + } + + + @Override + public ApiServiceVo queryById(Long id,String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + ApiVersionVo maxApiVersionVo = getMaxVersion(apiServiceVo.getId()); + List userTokenManagerVos = apiServiceTokenManagerDao.queryByApplyUserAndVersionId(userName,maxApiVersionVo.getId()); + if(userTokenManagerVos.size()>0) { + // query param + if (apiServiceVo != null) { + apiServiceVo.setParams(apiServiceParamDao.queryByVersionId(maxApiVersionVo.getId())); + apiServiceVo.setUserToken(userTokenManagerVos.get(0).getToken()); + apiServiceVo.setLatestVersionId(maxApiVersionVo.getId()); + } + }else { + return null; + } + return apiServiceVo; + } + + + @Override + public List queryAllTags(String userName,Integer workspaceId) { + //todo 会有历史版本的tag + List tags = apiServiceDao.queryAllTags(userName,workspaceId); + List tagList= + tags.stream().filter(tag->!StringUtils.isEmpty(tag)).map(tag -> Splitter.on(",").splitToList(tag)).flatMap(List::stream).distinct() + .collect(Collectors.toList()); + return tagList; + } + + + @Override + public ApiServiceVo queryByScriptPath(@Param("scriptPath") String scriptPath) { + List apiServiceList = apiServiceDao.queryByScriptPath(scriptPath); + ApiServiceVo latestApiService = apiServiceList.stream().max(Comparator.comparing(ApiServiceVo::getModifyTime)).orElse(null); + if(null == latestApiService){ + return null; + } + List apiVersionVos =apiServiceVersionDao.queryApiVersionByApiServiceId(latestApiService.getId()); + ApiVersionVo maxVersion =apiVersionVos.stream().max(Comparator.comparing(ApiVersionVo::getVersion)).orElse(null); + // query param + if (latestApiService != null && null != maxVersion ) { + latestApiService.setParams(apiServiceParamDao.queryByVersionId(maxVersion.getId())); + latestApiService.setLatestVersionId(maxVersion.getId()); + } + return latestApiService; + } + + + + + + + @Override + public Integer queryCountByPath(String scriptPath, String path) { + return apiServiceDao.queryCountByPath(scriptPath, path); + } + + @Override + public Integer queryCountByName(String name) { + return apiServiceDao.queryCountByName(name); + } + + @Override + public Boolean enableApi(Long id,String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + if(!checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue())){ + LOG.error("api service check workspace error"); + return false; + } + if(apiServiceVo.getCreator().equals(userName)) { + + Integer updateCount = apiServiceDao.enableApi(id); + List targetApiVersionList = apiServiceVersionDao.queryApiVersionByApiServiceId(id); + ApiVersionVo maxTargetApiVersionVo = targetApiVersionList.stream().max(Comparator.comparing(ApiVersionVo::getVersion)).orElse(null); + + apiServiceTokenManagerDao.enableTokenStatusByVersionId(maxTargetApiVersionVo.getId()); + apiServiceVersionDao.updateApiVersionStatusById(maxTargetApiVersionVo.getId(), 1); + return updateCount > 0; + }else { + return false; + } + } + + @Override + public Boolean disableApi(Long id,String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + if(!checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue())){ + LOG.error("api service check workspace error"); + return false; + } + if(apiServiceVo.getCreator().equals(userName)) { + Integer updateCount = apiServiceDao.disableApi(id); + apiServiceTokenManagerDao.disableTokenStatusByApiId(id); + apiServiceVersionDao.updateAllApiVersionStatusByApiServiceId(id, 0); + return updateCount > 0; + }else { + return false; + } + } + + + @Override + public Boolean deleteApi(Long id,String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + if(!checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue())){ + LOG.error("api service check workspace error"); + return false; + } + if(apiServiceVo.getCreator().equals(userName)) { + Integer updateCount = apiServiceDao.deleteApi(id); + apiServiceTokenManagerDao.disableTokenStatusByApiId(id); + apiServiceVersionDao.updateAllApiVersionStatusByApiServiceId(id, 0); + return updateCount > 0; + }else { + return false; + } + } + + @Override + public Boolean updateComment(Long id, String comment, String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + if(!checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue())){ + LOG.error("api service check workspace error"); + return false; + } + if(apiServiceVo.getCreator().equals(userName)) { + Integer updateCount = apiServiceDao.updateApiServiceComment(id,comment); + return updateCount > 0; + }else { + return false; + } + } + + private Map uploadBml(String userName, String scriptPath, Map metadata, String scriptContent) { + try { + ScriptFsWriter writer = StorageScriptFsWriter.getScriptFsWriter(new FsPath(scriptPath), Consts.UTF_8.toString(), null); + List variableList=null; + if(metadata.entrySet().size() >0) { + Variable[] v = VariableParser.getVariables(metadata); + variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.value())).collect(Collectors.toList()); + + } + if(variableList!=null) { + writer.addMetaData(new ScriptMetaData(variableList.toArray(new Variable[0]))); + }else { + writer.addMetaData(null); + } + writer.addRecord(new ScriptRecord(scriptContent)); + InputStream inputStream = writer.getInputStream(); + // 新增文件 + BmlUploadResponse resource = client.uploadResource(userName, scriptPath, inputStream); + if (!resource.isSuccess()) { + throw new IOException("upload bml error"); + } + Map result = new HashMap<>(); + result.put("resourceId", resource.resourceId()); + result.put("version", resource.version()); + return result; + } catch (IOException e) { + LOG.error("upload bml error", e); + throw new RuntimeException(e); + } + } + + private Map updateBml(String userName, String resourceId, String scriptPath, Map metadata, String scriptContent) { + try { + ScriptFsWriter writer = StorageScriptFsWriter.getScriptFsWriter(new FsPath(scriptPath), Consts.UTF_8.toString(), null); + Variable[] v = VariableParser.getVariables(metadata); + List variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.value())).collect(Collectors.toList()); + writer.addMetaData(new ScriptMetaData(variableList.toArray(new Variable[0]))); + writer.addRecord(new ScriptRecord(scriptContent)); + InputStream inputStream = writer.getInputStream(); + + // 更新文件 + BmlUpdateResponse resource = client.updateResource(userName, resourceId, "", inputStream); + if (!resource.isSuccess()) { + throw new IOException("update bml error"); + } + Map result = new HashMap<>(); + result.put("resourceId", resource.resourceId()); + result.put("version", resource.version()); + return result; + } catch (IOException e) { + LOG.error("update bml error", e); + throw new RuntimeException(e); + } + } + + private void removeBml(String userName, String resourceId) { + try { + LOG.info("delete bml resource: userName: " + userName + ", resourceId: " + resourceId); + client.deleteResource(userName, resourceId); + } catch (Exception e) { + LOG.error("remove bml error", e); + } + } + + + public SQLMetadataInfoCheckStatus sendApprovalToDM(String user, ApiServiceVo apiService, ApiVersionVo apiVersionVo) throws Exception { + + String approvalNo = UUIDGenerator.genUUID(); + apiVersionVo.setAuthId(approvalNo); + return SQLMetadataInfoCheckStatus.LEGAL; + } + + + + + public void genTokenForPublisher(ApiServiceVo apiService,Long apiVersionId) throws ApiServiceTokenException { + + List tokenManagerVoList = genTokenForAccessApi(apiService,apiVersionId); + SaveTokenEnum res = tokenAuth.saveTokensToDb(tokenManagerVoList, ApiCommonConstant.DEFAULT_APPROVAL_NO); + } + + /** + * 为申请用户和自己生成访问token + * @param apiService + * @param apiVersionId + * @return + */ + private List genTokenForAccessApi(ApiServiceVo apiService,Long apiVersionId){ + List tokenManagerVoList = new ArrayList<>(); + List userNamesTmp = Arrays.asList(apiService.getApprovalVo().getApplyUser().split(",")); + List userNames = new ArrayList<>(userNamesTmp); + userNames.add(apiService.getCreator()); + userNames.stream().forEach(userName ->{ + TokenManagerVo tokenManagerVo = new TokenManagerVo(); + tokenManagerVo.setApiId(apiService.getId()); + tokenManagerVo.setApiVersionId(apiVersionId); + tokenManagerVo.setApplyTime(Calendar.getInstance().getTime()); + tokenManagerVo.setDuration(365L); + tokenManagerVo.setReason("create api service"); + tokenManagerVo.setStatus(1); + tokenManagerVo.setIpWhitelist(""); + tokenManagerVo.setCaller("scriptis"); + tokenManagerVo.setUser(userName); + tokenManagerVo.setPublisher(apiService.getCreator()); + + ApiServiceToken apiServiceToken = new ApiServiceToken(); + apiServiceToken.setApplyUser(userName); + apiServiceToken.setPublisher(apiService.getCreator()); + apiServiceToken.setApplyTime(tokenManagerVo.getApplyTime()); + apiServiceToken.setApiServiceId(tokenManagerVo.getApiId()); + + tokenManagerVo.setToken(JwtManager.createToken(userName,apiServiceToken,tokenManagerVo.getDuration())); + + tokenManagerVo.setApplySource(ApiCommonConstant.DEFAULT_APPROVAL_NO); + tokenManagerVoList.add(tokenManagerVo); + }); + + return tokenManagerVoList; + } + + + @Override + public ApiVersionVo getMaxVersion(long serviceId){ + List apiVersionVoList = apiServiceVersionDao.queryApiVersionByApiServiceId(serviceId); + ApiVersionVo maxApiVersionVo = apiVersionVoList.stream().max(Comparator.comparing(ApiVersionVo::getVersion)).orElse(null); + return maxApiVersionVo; + } + + @Override + public boolean checkUserWorkspace(String userName,Integer workspaceId){ + //todo cache user workspaceIds + return true; +// String workspaceIds = ExecuteCodeHelper.getUserWorkspaceIds(userName,ujesClient); +// if(Arrays.stream(workspaceIds.split(",")).map(Integer::valueOf).collect(Collectors.toList()).contains(workspaceId)){ +// return true; +// }else { +// return false; +// } + + } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java new file mode 100644 index 000000000..fbfb1f803 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java @@ -0,0 +1,546 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.service.impl; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalCause; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceJob; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.bo.LinkisExecuteResult; +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.dss.apiservice.core.constant.ParamType; +import com.webank.wedatasphere.dss.apiservice.core.constant.ParamTypeEnum; +import com.webank.wedatasphere.dss.apiservice.core.constant.RequireEnum; +import com.webank.wedatasphere.dss.apiservice.core.dao.*; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiExecuteException; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.execute.ApiServiceExecuteJob; +import com.webank.wedatasphere.dss.apiservice.core.execute.DefaultApiServiceJob; +import com.webank.wedatasphere.dss.apiservice.core.execute.ExecuteCodeHelper; +import com.webank.wedatasphere.dss.apiservice.core.execute.LinkisJobSubmit; +import com.webank.wedatasphere.dss.apiservice.core.jdbc.DatasourceService; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiService; +import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil; +import com.webank.wedatasphere.dss.apiservice.core.util.SQLCheckUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; +//import com.webank.wedatasphere.dss.oneservice.core.jdbc.JdbcUtil; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceRuntimeException; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiServiceQueryService; +import com.webank.wedatasphere.dss.apiservice.core.util.AssertUtil; +import com.webank.wedatasphere.dss.apiservice.core.util.ModelMapperUtil; +//import com.webank.wedatasphere.dss.oneservice.core.vo.*; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiServiceVo; +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.bml.protocol.BmlDownloadResponse; +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.storage.source.FileSource; +import org.apache.linkis.storage.source.FileSource$; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.math3.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.support.JdbcUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedCaseInsensitiveMap; +import scala.Tuple3; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.io.InputStream; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toMap; + + +@Service +public class ApiServiceQueryServiceImpl implements ApiServiceQueryService { + private static final Logger LOG = LoggerFactory.getLogger(ApiServiceQueryServiceImpl.class); + + + Map runJobs = new HashMap<>(); + + /** + * key:resourceId+version + * value:bml + */ + private static Cache>> bmlCache = CacheBuilder.newBuilder() + .expireAfterWrite(6, TimeUnit.HOURS) + .maximumSize(10000) + .removalListener((notification) -> { + if (notification.getCause() == RemovalCause.SIZE) { + LOG.warn("bml缓存容量不足,移除key:" + notification.getKey()); + } + }) + .build(); + + /** + * key:resourceId+version + * value:configParam + */ + private static Cache> configParamCache = CacheBuilder.newBuilder() + .expireAfterWrite(6, TimeUnit.HOURS) + .maximumSize(10000) + .removalListener((notification) -> { + if (notification.getCause() == RemovalCause.SIZE) { + LOG.warn("configParamCache缓存容量不足,移除key:" + notification.getKey()); + } + }) + .build(); + + /** + * key:datasourceMap + * value:jdbc连接信息 + */ + private static Cache, Tuple3> datasourceCache = CacheBuilder.newBuilder() + .expireAfterWrite(1, TimeUnit.MINUTES) + .maximumSize(2000) + .removalListener((notification) -> { + if (notification.getCause() == RemovalCause.SIZE) { + LOG.warn("datasource缓存容量不足,移除key:" + notification.getKey()); + } + }) + .build(); + + @Autowired + private ApiServiceDao apiServiceDao; + + @Autowired + private ApiServiceParamDao apiServiceParamDao; + + @Autowired + private DatasourceService datasourceService; + + @Autowired + private ApiServiceVersionDao apiServiceVersionDao; + + @Autowired + private ApiServiceTokenManagerDao apiServiceTokenManagerDao; + + + @Autowired + private ApiService apiService; + + + @Autowired + private ApiServiceAccessDao apiServiceAccessDao; + + /** + * Bml client + */ + private BmlClient client; + + @PostConstruct + public void init() { + LOG.info("build client start ======"); + client = BmlClientFactory.createBmlClient(); + LOG.info("build client end ======="); + } + + @Override + public LinkisExecuteResult query(String path, + Map reqParams, + String moduleName, + String httpMethod, + ApiServiceToken tokenDetail, + String loginUser) { + // 根据path查询resourceId和version + + // 得到metadata + + // 执行查询 + //path 必须唯一 + ApiServiceVo apiServiceVo = apiServiceDao.queryByPath(path); + if(null == apiServiceVo){ + throw new ApiServiceRuntimeException("根据脚本路径未匹配到数据服务!"); + } + if(!apiService.checkUserWorkspace(loginUser,apiServiceVo.getWorkspaceId().intValue())){ + throw new ApiServiceRuntimeException("用户工作空间检查不通过!"); + } + if(!apiServiceVo.getId().equals(tokenDetail.getApiServiceId())){ + throw new ApiServiceRuntimeException("用户token中服务ID不匹配!"); + } + + ApiVersionVo maxApiVersionVo =apiService.getMaxVersion(apiServiceVo.getId()); + + AssertUtil.notNull(apiServiceVo, "接口不存在,path=" + path); + AssertUtil.isTrue(StringUtils.equals(httpMethod, apiServiceVo.getMethod().toUpperCase()), + "该接口不支持" + httpMethod + "请求,请用" + apiServiceVo.getMethod() + "请求"); + AssertUtil.isTrue(1 == apiServiceVo.getStatus(), "接口已禁用"); + AssertUtil.notNull(maxApiVersionVo, "未找到最新的版本,path=" + path); + + try { + Pair> collect = queryBml(apiServiceVo.getCreator(), maxApiVersionVo.getBmlResourceId(), + maxApiVersionVo.getBmlVersion(), apiServiceVo.getScriptPath()); + String executeCode = collect.getSecond().get(0)[0]; + + Map variable = (Map) ((Map) collect.getFirst()).get("variable"); + + //没有传入的参数,使用默认值 + Map paramTypes = queryConfigParam(apiServiceVo.getId(), maxApiVersionVo.getVersion()); + if (variable != null) { + variable.forEach((k, v) -> { + if (!reqParams.containsKey(k)) { + if (ParamType.number.equals(paramTypes.get(k))) { + reqParams.put(k, Integer.valueOf(v.toString())); + } else { + reqParams.put(k, v); + } + } + }); + } + + // 用户请求的参数值注入检查,排除token + for(String k: reqParams.keySet()){ + if(!k.equals(ApiServiceConfiguration.API_SERVICE_TOKEN_KEY.getValue()) + && SQLCheckUtil.doParamInjectionCheck(reqParams.get(k).toString())) { + // 如果注入直接返回null + LOG.warn("用户参数存在非法的关键字{}", reqParams.get(k).toString()); + return null; + } + } + + //数组类型,如果没有加单引号,自动添加 + reqParams.forEach((k,v) ->{ + if(ParamType.array.equals(paramTypes.get(k))){ + String sourceStr = v.toString(); + String targetStr =sourceStr; + sourceStr= sourceStr.replaceAll("(\n\r|\r\n|\r|\n)", ","); + sourceStr= sourceStr.replaceAll(",,", ","); + + if(!sourceStr.contains("\'")){ + targetStr= Arrays.stream(sourceStr.split(",")).map(s -> "\'" + s + "\'").collect(Collectors.joining(",")); + reqParams.put(k, targetStr); + }else { + reqParams.put(k, sourceStr); + } + } + }); + + +// AssertUtil.isTrue(MapUtils.isNotEmpty((Map) collect.getKey()), "数据源不能为空"); + + + ApiServiceExecuteJob job = new DefaultApiServiceJob(); + //sql代码封装成scala执行 + job.setCode(ExecuteCodeHelper.packageCodeToExecute(executeCode, maxApiVersionVo.getMetadataInfo())); + job.setEngineType(apiServiceVo.getType()); + job.setRunType("scala"); + //不允许创建用户自己随意代理执行,创建用户只能用自己用户执行 + //如果需要代理执行可以在这里更改用户 + job.setUser(loginUser); + + job.setParams(null); + job.setRuntimeParams(reqParams); + job.setScriptePath(apiServiceVo.getScriptPath()); + UJESClient ujesClient = LinkisJobSubmit.getClient(paramTypes); + + //记录用户Api被执行信息 + ApiAccessVo apiAccessVo = new ApiAccessVo(); + apiAccessVo.setUser(loginUser); + apiAccessVo.setApiPublisher(apiServiceVo.getCreator()); + apiAccessVo.setApiServiceName(apiServiceVo.getName()); + apiAccessVo.setApiServiceId(apiServiceVo.getId()); + apiAccessVo.setApiServiceVersionId(maxApiVersionVo.getId()); + apiAccessVo.setProxyUser(job.getUser()); + apiAccessVo.setAccessTime(DateUtil.getNow()); + apiServiceAccessDao.addAccessRecord(apiAccessVo); + + + JobExecuteResult jobExecuteResult = LinkisJobSubmit.execute(job,ujesClient); + + //记录执行任务用户和代理用户关系,没有代理用户的统一设置为登录用户 + ApiServiceJob apiServiceJob = new ApiServiceJob(); + apiServiceJob.setSubmitUser(loginUser); + apiServiceJob.setProxyUser(job.getUser()); + apiServiceJob.setJobExecuteResult(jobExecuteResult); + runJobs.put(jobExecuteResult.getTaskID(),apiServiceJob); + + + LinkisExecuteResult linkisExecuteResult = new LinkisExecuteResult(jobExecuteResult.getTaskID(), jobExecuteResult.getExecID()); + return linkisExecuteResult; + } catch (IOException e) { + throw new ApiServiceRuntimeException(e.getMessage(), e); + } + } + + + + + @Override + public ApiServiceVo queryByVersionId(String userName,Long versionId) throws ApiServiceQueryException { + ApiVersionVo apiVersionVo = apiServiceVersionDao.queryApiVersionByVersionId(versionId); + ApiServiceVo apiServiceVo = apiServiceDao.queryById(apiVersionVo.getApiId()); + //授权后才可以查看内容 + List userTokenManagerVos = apiServiceTokenManagerDao.queryByApplyUserAndVersionId(userName,versionId); + if(userTokenManagerVos.size()>0) { + try { + Pair> collect = queryBml(apiServiceVo.getCreator(), apiVersionVo.getBmlResourceId(), + apiVersionVo.getBmlVersion(), apiServiceVo.getScriptPath()); + String executeCode = collect.getSecond().get(0)[0]; + apiServiceVo.setContent(executeCode); + + } catch (IOException e) { + throw new ApiServiceQueryException(800002, "查询数据服务API内容异常"); + } + apiServiceVo.setScriptPath(apiVersionVo.getSource()); + return apiServiceVo; + }else { + + throw new ApiServiceQueryException(800003, "没有权限查看数据服务API内容,请先提单授权"); + } + } + + @Override + public List queryParamList(String scriptPath, Long versionId) { + ApiVersionVo targetApiVersionVo = apiServiceVersionDao.queryApiVersionByVersionId(versionId); + + ApiServiceVo apiServiceVo=apiServiceDao.queryById(targetApiVersionVo.getApiId()); + + AssertUtil.notNull(apiServiceVo, "接口不存在,path=" + scriptPath); + + AssertUtil.notNull(targetApiVersionVo, "目标参数版本不存在,path=" + scriptPath+",version:"+versionId); + + // todo~! + List paramVoList = apiServiceParamDao.queryByVersionId(targetApiVersionVo.getId()); + + + List queryParamVoList = new ArrayList<>(); + + Map paramMap = paramVoList.stream() + .collect(Collectors.toMap(ParamVo::getName, k -> k, (k, v) -> k)); + Map variableMap = getVariable(apiServiceVo,versionId); + paramMap.keySet() + .forEach(keyItem -> { + ParamVo paramVo = paramMap.get(keyItem); + QueryParamVo queryParamVo = ModelMapperUtil.strictMap(paramVo, QueryParamVo.class); + queryParamVo.setTestValue(variableMap.containsKey(keyItem) ? variableMap.get(keyItem).toString() : ""); + queryParamVo.setRequireStr(RequireEnum.getEnum(paramVo.getRequired()).getName()); + queryParamVo.setType(paramVo.getType()); + + queryParamVoList.add(queryParamVo); + }); + + return queryParamVoList; + } + + @Override + public List queryApiVersionById(Long serviceId) { + List apiVersionVoList = apiServiceVersionDao.queryApiVersionByApiServiceId(serviceId); + return apiVersionVoList; + } + + private Map getVariable(ApiServiceVo apiServiceVo,Long versionId) { + Map variableMap = null; + ApiVersionVo apiVersionVo = apiServiceVersionDao.queryApiVersionByVersionId(versionId); + if(null != apiServiceVo) { + try { + Pair> collect = queryBml(apiServiceVo.getCreator(), apiVersionVo.getBmlResourceId(), + apiVersionVo.getBmlVersion(), apiServiceVo.getScriptPath()); + + variableMap = (Map) ((Map) collect.getFirst()).get("variable"); + } catch (IOException e) { + throw new ApiServiceRuntimeException(e.getMessage(), e); + } + } + return null == variableMap ? Collections.EMPTY_MAP : variableMap; + } + + private Pair> queryBml(String userName, String resourceId, String version, + String scriptPath) throws IOException { + String key = String.join("-", resourceId, version); + Pair> collect = bmlCache.getIfPresent(key); + + if (collect == null) { + synchronized (this) { + collect = bmlCache.getIfPresent(key); + if (collect == null) { + BmlDownloadResponse resource; + if (version == null) { + resource = client.downloadResource(userName, resourceId, null); + } else { + resource = client.downloadResource(userName, resourceId, version); + } + + AssertUtil.isTrue(resource.isSuccess(), "查询bml错误"); + + InputStream inputStream = resource.inputStream(); + + try (FileSource fileSource = FileSource$.MODULE$.create(new FsPath(scriptPath), inputStream)) { + //todo 数组取了第一个 + collect = fileSource.collect()[0]; + bmlCache.put(key, collect); + } + } + } + } + + + return collect; + } + + private Map queryConfigParam(long apiId, String version) { + String key = String.join("-", apiId + "", version); + Map collect = configParamCache.getIfPresent(key); + + if (collect == null) { + synchronized (this) { + collect = configParamCache.getIfPresent(key); + if (collect == null) { + List apiVersionVoList = apiServiceVersionDao.queryApiVersionByApiServiceId(apiId); + ApiVersionVo apiVersionVo = apiVersionVoList.stream().filter(apiVersionVoTmp -> apiVersionVoTmp.getVersion().equals(version)).findFirst().orElse(null); + + collect = apiServiceParamDao.queryByVersionId(apiVersionVo.getId()) + .stream() + .collect(toMap(ParamVo::getName, ParamVo::getType)); + configParamCache.put(key, collect); + } + } + } + + return collect; + } + + + + +// private Tuple3 getDatasourceInfo(final Map datasourceMap) { +// Tuple3 tuple3 = datasourceCache.getIfPresent(datasourceMap); +// +// if (tuple3 == null) { +// synchronized (this) { +// tuple3 = datasourceCache.getIfPresent(datasourceMap); +// if (tuple3 == null) { +// tuple3 = JdbcUtil.getDatasourceInfo(datasourceMap); +// datasourceCache.put(datasourceMap, tuple3); +// } +// } +// } +// +// return tuple3; +// } + +// private List> executeJob(String executeCode, +// Object datasourceMap, Map params) { +// +//// Tuple3 tuple3 = getDatasourceInfo((Map) datasourceMap); +//// final String jdbcUrl = tuple3._1().toString(); +//// final String username = tuple3._2().toString(); +//// final String password = tuple3._3().toString(); +// +//// NamedParameterJdbcTemplate namedParameterJdbcTemplate = datasourceService.getNamedParameterJdbcTemplate(jdbcUrl, username, password); +// +// String namedSql = genNamedSql(executeCode, params); +// +//// return namedParameterJdbcTemplate.query(namedSql, new MapSqlParameterSource(params), new ColumnAliasMapRowMapper()); +// +// } + + private static String genNamedSql(String executeCode, Map params) { + // 没有参数,无需生成namedSql + if (MapUtils.isEmpty(params)) { + return executeCode; + } + + for (String paramName : params.keySet()) { + for (String $name : new String[]{"'${" + paramName + "}'", "${" + paramName + "}", "\"${" + paramName + "}\""}) { + if (executeCode.contains($name)) { + executeCode = StringUtils.replace(executeCode, $name, ":" + paramName); + break; + } + } + } + + return executeCode; + } + + + public static class ColumnAliasMapRowMapper implements RowMapper> { + @Override + public Map mapRow(ResultSet rs, int rowNum) throws SQLException { + ResultSetMetaData rsmd = rs.getMetaData(); + int columnCount = rsmd.getColumnCount(); + Map mapOfColValues = createColumnMap(columnCount); + Map mapOfColSuffix = new LinkedCaseInsensitiveMap<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + String key = getColumnKey(JdbcUtils.lookupColumnName(rsmd, i)); + if (mapOfColValues.containsKey(key)) { + if (!mapOfColSuffix.containsKey(key)) { + mapOfColSuffix.put(key, 1); + } else { + mapOfColSuffix.put(key, mapOfColSuffix.get(key) + 1); + } + + key = key + "_" + mapOfColSuffix.get(key); + } + + Object obj = getColumnValue(rs, i); + mapOfColValues.put(key, obj); + } + return mapOfColValues; + } + + protected Map createColumnMap(int columnCount) { + return new LinkedCaseInsensitiveMap<>(columnCount); + } + + protected String getColumnKey(String columnName) { + return columnName; + } + + protected Object getColumnValue(ResultSet rs, int index) throws SQLException { + return JdbcUtils.getResultSetValue(rs, index); + } + + } + + @Override + public ApiServiceJob getJobByTaskId(String taskId){ + ApiServiceJob apiServiceJob=runJobs.get(taskId); + return apiServiceJob; + } + + + private static String getRunTypeFromScriptsPath(String scriptsPath) { + + String res = "sql"; + String fileFlag = scriptsPath.substring(scriptsPath.lastIndexOf(".") + 1); + switch (fileFlag) { + case "sh": + res = "shell"; + break; + case "py": + res= "pyspark"; + break; + default: + res = fileFlag; + break; + } + return res; + + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/TokenQueryServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/TokenQueryServiceImpl.java new file mode 100644 index 000000000..f5dc98e93 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/TokenQueryServiceImpl.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service.impl; + +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceTokenManagerDao; +import com.webank.wedatasphere.dss.apiservice.core.service.TokenQueryService; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import com.webank.wedatasphere.dss.apiservice.core.bo.TokenQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class TokenQueryServiceImpl implements TokenQueryService { + private static final Logger LOG = LoggerFactory.getLogger(TokenQueryServiceImpl.class); + + @Autowired + ApiServiceTokenManagerDao apiServiceTokenManagerDao; + + @Override + public PageInfo query(TokenQuery tokenQuery) { + PageHelper.startPage(tokenQuery.getCurrentPage(), tokenQuery.getPageSize()); + List tokenList = apiServiceTokenManagerDao.query(tokenQuery); + LOG.info("token查询的结果列表大小为{}", tokenList.size()); + PageInfo pageInfo = new PageInfo<>(tokenList); + return pageInfo; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/DataMapTokenImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/DataMapTokenImpl.java new file mode 100644 index 000000000..bb32448d8 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/DataMapTokenImpl.java @@ -0,0 +1,109 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.webank.wedatasphere.dss.apiservice.core.constant.ApiCommonConstant; +import com.webank.wedatasphere.dss.apiservice.core.constant.SaveTokenEnum; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceTokenManagerDao; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceTokenException; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.*; + + + +@Component +public class DataMapTokenImpl implements TokenAuth { + + private static final Logger logger = LoggerFactory.getLogger(DataMapTokenImpl.class); + + + @Autowired + ApiServiceTokenManagerDao apiServiceTokenManagerDao; + + @Override + public SaveTokenEnum saveTokensToDb(List tokenManagerVos, String approvalNo) throws ApiServiceTokenException { + boolean isEmptyToken = tokenManagerVos.stream().filter(tokenManagerVo -> StringUtils.isEmpty(tokenManagerVo.getToken())).count()>0; + if(checkDuplicateAuth(approvalNo) && approvalNo != ApiCommonConstant.DEFAULT_APPROVAL_NO){ +// throw new ApiServiceTokenException(800001,"ApprovalNo has been repeatedly authorized "); + return SaveTokenEnum.DUPLICATE; + }else if(isEmptyToken){ + throw new ApiServiceTokenException(800001,"Failed to save to db for Some token is empty"); + } + else { + + try { + apiServiceTokenManagerDao.insertList(tokenManagerVos); + } catch (Exception e) { + logger.error("Batch save token to db failed", e); + return SaveTokenEnum.FAILED; + } + } + + return SaveTokenEnum.SUCCESS; + } + + private boolean checkDuplicateAuth(String approvalNo){ + if(apiServiceTokenManagerDao.queryApprovalNo(approvalNo) > 0){ + return true; + }else { + return false; + } + } + + @Override + public List genTokenRecord(ApprovalVo approvalVo){ + + List tokenManagerVoList = new ArrayList<>(); + Arrays.stream(approvalVo.getApplyUser().split(",")).forEach(tempUser ->{ + + TokenManagerVo tmpToken = new TokenManagerVo(); + tmpToken.setApiId(approvalVo.getApiId()); + tmpToken.setApplyTime(new Date()); + tmpToken.setDuration(365L); + tmpToken.setReason("approval token auth"); + tmpToken.setStatus(1); + tmpToken.setIpWhitelist(""); + tmpToken.setCaller("scripts"); + tmpToken.setUser(tempUser); + tmpToken.setPublisher(approvalVo.getCreator()); + tmpToken.setApiVersionId(approvalVo.getApiVersionId()); + + ApiServiceToken apiServiceToken = new ApiServiceToken(); + apiServiceToken.setApplyUser(tempUser); + apiServiceToken.setPublisher(approvalVo.getCreator()); //todo creator + apiServiceToken.setApplyTime(tmpToken.getApplyTime()); + apiServiceToken.setApiServiceId(approvalVo.getApiId()); + + tmpToken.setToken(JwtManager.createToken(tempUser,apiServiceToken,tmpToken.getDuration())); + //审批单号 + tmpToken.setApplySource(approvalVo.getApprovalNo()); + tokenManagerVoList.add(tmpToken); + + + }); + + return tokenManagerVoList; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java new file mode 100644 index 000000000..cb0bea3c7 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java @@ -0,0 +1,119 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.sun.jersey.core.util.Base64; +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import org.apache.linkis.common.conf.CommonVars$; +import io.jsonwebtoken.Claims; +import io.jsonwebtoken.ExpiredJwtException; +import io.jsonwebtoken.Jws; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.MalformedJwtException; +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.SignatureException; +import io.jsonwebtoken.UnsupportedJwtException; +import io.jsonwebtoken.impl.crypto.MacProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.TemporalAmount; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + + +public class JwtManager { + private static final Logger LOG = LoggerFactory.getLogger(JwtManager.class); + + private static final String CLAIM_ROLE = "role"; + + private static final SignatureAlgorithm SIGNATURE_ALGORITHM = SignatureAlgorithm.HS256; + private static final SecretKey SECRET_KEY = MacProvider.generateKey(SIGNATURE_ALGORITHM); + private static final TemporalAmount TOKEN_VALIDITY = getTokenHour(); + private static final String JWT_SECERT = ApiServiceConfiguration.DSS_API_TOKEN_SECRET_ID.getValue(); + + public final static TemporalAmount getTokenHour() { + Properties conf = CommonVars$.MODULE$.properties(); + if (null != conf) { + Object hour = conf.get("ujes.token.valid.hour"); + if (null == hour) { + return Duration.ofHours(8L); + } else { + return Duration.ofHours(Long.parseLong(hour.toString())); + } + } else { + return Duration.ofDays(365); + } + + } + + public static SecretKey generalKey() { + byte[] encodedKey = Base64.decode(JWT_SECERT); + SecretKey key = new SecretKeySpec(encodedKey, 0, encodedKey.length, "AES"); + return key; + } + + /** + * Builds a JWT with the given subject and role and returns it as a JWS signed compact String. + */ + public static String createToken(final String applyUser, ApiServiceToken apiServiceToken, Long duration) { + final String role = "developer"; + final Instant now = Instant.now(); + final Date expiryDate = Date.from(now.plus(Duration.ofDays(duration))); + Map tokenDetailMap = new HashMap(); + tokenDetailMap.put("tokenDetail", apiServiceToken); + return Jwts.builder() + .setSubject(applyUser) + .claim(CLAIM_ROLE, role) + .setClaims(tokenDetailMap) + .setExpiration(expiryDate) + .setIssuedAt(Date.from(now)) + .signWith(SIGNATURE_ALGORITHM, generalKey()) + .compact(); + } + + /** + * Parses the given JWS signed compact JWT, returning the claims. + * If this method returns without throwing an exception, the token can be trusted. + */ + public static ApiServiceToken parseToken(final String compactToken) + throws ExpiredJwtException, + UnsupportedJwtException, + MalformedJwtException, + SignatureException, + IllegalArgumentException { + + Jws jws = Jwts.parser().setSigningKey(generalKey()).parseClaimsJws(compactToken); + + Map tokenDetail = (Map) jws.getBody().get("tokenDetail"); + ApiServiceToken apiServiceToken = new ApiServiceToken(); + apiServiceToken.setApiServiceId(Long.valueOf(tokenDetail.get("apiServiceId").toString())); + apiServiceToken.setApplyTime(null); + apiServiceToken.setPublisher(tokenDetail.get("publisher").toString()); + apiServiceToken.setApplyUser(tokenDetail.get("applyUser").toString()); + + return apiServiceToken; + } + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenAuth.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenAuth.java new file mode 100644 index 000000000..b7385ea8c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenAuth.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.webank.wedatasphere.dss.apiservice.core.constant.SaveTokenEnum; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceTokenException; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; + +import java.util.List; + + +public interface TokenAuth { + + + /** + * batch save token to db + * @param tokenManagerVos + * @return + */ + SaveTokenEnum saveTokensToDb(List tokenManagerVos, String approvalNo) throws ApiServiceTokenException; + + List genTokenRecord(ApprovalVo approvalVo); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java new file mode 100644 index 000000000..95203fc07 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceTokenManagerDao; +import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +@Component +@EnableScheduling +public class TokenStatusCheckerTask { + private static final Logger LOG = LoggerFactory.getLogger(TokenStatusCheckerTask.class); + + @Autowired + ApiServiceTokenManagerDao atmd; + + @Scheduled(cron = "0 0/5 * * * ?") + public void doTokenStatusCheckTask() { + // 查询启用状态的token + List tokenManagerVos = atmd.queryTokenByStatus(1); + if (null != tokenManagerVos) { + for (TokenManagerVo tmv : tokenManagerVos) { + Date applyTime = tmv.getApplyTime(); + Date nowTime = Calendar.getInstance().getTime(); + Calendar cal = Calendar.getInstance(); + cal.setTime(applyTime); + cal.add(Calendar.DATE, tmv.getDuration().intValue()); + Date endTime = cal.getTime(); + if (endTime.compareTo(nowTime) < 0) { + LOG.warn("token id:" + tmv.getId() + " 已经过期!"); + atmd.disableTokenStatus(tmv.getId()); + } + } + } + } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ApiUtils.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ApiUtils.java new file mode 100644 index 000000000..972272be1 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ApiUtils.java @@ -0,0 +1,111 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.util; + +import com.webank.wedatasphere.dss.apiservice.core.exception.*; +import com.webank.wedatasphere.dss.apiservice.core.vo.MessageVo; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.validation.ConstraintViolationException; +import javax.ws.rs.core.Response; + + +public class ApiUtils { + + private static final Logger LOG = LoggerFactory.getLogger(ApiUtils.class); + + /** + * @param tryOperation operate function + * @param failMessage message + */ + public static Message doAndResponse(TryOperation tryOperation, String method, String failMessage) { + try { + Message message = tryOperation.operateAndGetMessage(); + return setMethod(message, method); + } catch (WarnException e) { + LOG.error("api error, method: " + method, e); + return setMethod(Message.warn(e.getMessage()), method); + } catch (AssertException | ApiExecuteException | ApiServiceQueryException e) { + LOG.error("api error, method: " + method, e); + return setMethod(Message.error(e.getMessage()), method); + } catch (Exception e) { + LOG.error("api error, method: " + method, e); + return Message.error(ExceptionUtils.getRootCauseMessage(e)); + } + } + + /** + * @param tryOperation operate function + */ + public static Response doAndResponse(Operation tryOperation) { + Object msg = null; + try { + msg = tryOperation.operateAndGetMessage(); + return Response.ok(msg).build(); + } catch (ConstraintViolationException e) { + LOG.error("api error ", e); + return new BeanValidationExceptionMapper().toResponse(e); + } catch (WarnException e) { + LOG.error("api error ", e); + return Response.ok(setMsg("系统异常")).build(); + } catch (AssertException e) { + LOG.error("api error ", e); + return Response.ok(setMsg(e.getMessage())).build(); + }catch (ApiServiceRuntimeException e){ + LOG.error("api error ", e); + return Response.ok(setMsg(e.getMessage())).build(); + } + catch (Exception e) { + LOG.error("api error ", e); + return Response.ok(setMsg(String.valueOf(e.getCause()))).build(); + } + } + + public static Object setMsg(String message) { + MessageVo messageVo = new MessageVo(); + messageVo.setMessage(message); + + return messageVo; + } + + private static Message setMethod(Message message, String method) { + message.setMethod(method); + return message; + } + + @FunctionalInterface + public interface TryOperation { + + /** + * Operate method + */ + Message operateAndGetMessage() throws Exception; + } + + @FunctionalInterface + public interface Operation { + + /** + * Operate method + */ + Object operateAndGetMessage() throws Exception; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/AssertUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/AssertUtil.java new file mode 100644 index 000000000..2ade7601d --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/AssertUtil.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.util; + + +import com.webank.wedatasphere.dss.apiservice.core.exception.AssertException; + + +public class AssertUtil { + + public static void isTrue(boolean b, String message) { + if (!b) { + throw new AssertException(message); + } + } + + public static void isFalse(boolean b, String message) { + if (b) { + throw new AssertException(message); + } + } + + public static void notEmpty(String str, String message) { + if (str == null || str.isEmpty()) { + throw new AssertException(message); + } + } + + public static void notNull(Object str, String message) { + if (str == null) { + throw new AssertException(message); + } + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/DateUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/DateUtil.java new file mode 100644 index 000000000..5d5dcaa60 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/DateUtil.java @@ -0,0 +1,353 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.util; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + + +public class DateUtil { + + /** + * 英文简写(默认)如:2010-12-01 + */ + public static String FORMAT_MONTH = "yyyy-MM"; + /** + * 英文简写(默认)如:2010-12-01 + */ + public static String FORMAT_SHORT = "yyyy-MM-dd"; + + public static String FORMAT_SHORT_REPORT = "yyyy/MM/dd"; + /** + * 英文全称 如:2010-12-01 23:15:06 + */ + public static String FORMAT_LONG = "yyyy-MM-dd HH:mm:ss"; + public static String FORMAT_WITHOUT_SECOND = "yyyy-MM-dd HH:mm"; + + public static String FORMAT_MINUTE_REPORT = "yyyy/MM/dd HH:mm"; + public static String FORMAT_MINUTE = "yyyy-MM-dd HH:mm"; + /** + * 精确到毫秒的完整时间 如:yyyy-MM-dd HH:mm:ss.S + */ + public static String FORMAT_FULL = "yyyy-MM-dd HH:mm:ss.S"; + /** + * 中文简写 如:2010年12月01日 + */ + public static String FORMAT_SHORT_CN = "yyyy年MM月dd日"; + /** + * 中文全称 如:2010年12月01日 23时15分06秒 + */ + public static String FORMAT_LONG_CN = "yyyy年MM月dd日 HH时mm分ss秒"; + /** + * 精确到毫秒的完整中文时间 + */ + public static String FORMAT_FULL_CN = "yyyy年MM月dd日 HH时mm分ss秒SSS毫秒"; + + /** + * 获得默认的 date pattern + */ + public static String getDatePattern() { + return FORMAT_LONG; + } + + /** + * 根据预设格式返回当前日期 + * + * @return + */ + public static String getNow() { + return format(new Date()); + } + + /** + * 根据用户格式返回当前日期 + * + * @param format + * @return + */ + public static String getNow(String format) { + return format(new Date(), format); + } + + /** + * 使用预设格式格式化日期 + * + * @param date + * @return + */ + public static String format(Date date) { + return format(date, getDatePattern()); + } + + /** + * 使用用户格式格式化日期 + * + * @param date 日期 + * @param pattern 日期格式 + * @return + */ + public static String format(Date date, String pattern) { + String returnValue = ""; + if (date != null) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + returnValue = df.format(date); + } + return (returnValue); + } + + /** + * 使用预设格式提取字符串日期 + * + * @param strDate 日期字符串 + * @return + */ + public static Date parse(String strDate) { + return parse(strDate, getDatePattern()); + } + + /** + * 使用用户格式提取字符串日期 + * + * @param strDate 日期字符串 + * @param pattern 日期格式 + * @return + */ + public static Date parse(String strDate, String pattern) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + try { + return df.parse(strDate); + } catch (ParseException e) { + e.printStackTrace(); + return null; + } + } + + /** + * 在日期上增加数个整月 + * + * @param date 日期 + * @param n 要增加的月数 + * @return + */ + public static Date addMonth(Date date, int n) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.MONTH, n); + return cal.getTime(); + } + + /** + * 在日期上增加天数 + * + * @param date 日期 + * @param n 要增加的天数 + * @return + */ + public static Date addDay(Date date, int n) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.DATE, n); + return cal.getTime(); + } + + /** + * 获取时间戳 + */ + public static String getTimeString() { + SimpleDateFormat df = new SimpleDateFormat(FORMAT_FULL); + Calendar calendar = Calendar.getInstance(); + return df.format(calendar.getTime()); + } + + /** + * 获取日期年份 + * + * @param date 日期 + * @return + */ + public static String getYear(Date date) { + return format(date).substring(0, 4); + } + + /** + * 按默认格式的字符串距离今天的天数 + * + * @param date 日期字符串 + * @return + */ + public static int countDays(String date) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date)); + long t1 = c.getTime().getTime(); + return (int) (t / 1000 - t1 / 1000) / 3600 / 24; + } + + /** + * 按用户格式字符串距离今天的天数 + * + * @param date 日期字符串 + * @param format 日期格式 + * @return + */ + public static int countDaysAbs(String date, String format) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date, format)); + long t1 = c.getTime().getTime(); + return (int) Math.abs(t / 1000 - t1 / 1000) / 3600 / 24; + } + + /** + * 按用户格式字符串距离今天的天数 + * + * @param date 日期字符串 + * @param format 日期格式 + * @return + */ + public static int countDays(String date, String format) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date, format)); + long t1 = c.getTime().getTime(); + return (int) (t / 1000 - t1 / 1000) / 3600 / 24; + } + + public static String timeStrComplete(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return timeStr + lastStr; + } + return null; + } + + public static Timestamp str2Time(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return Timestamp.valueOf(timeStr + lastStr); + } + + return null; + } + + public static Timestamp str2TimeMax(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 23:59:59"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return Timestamp.valueOf(timeStr + lastStr); + } + + return null; + } + + public static Date str2DateTime(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return parse(timeStr + lastStr); + } + + return null; + } + + public static Date str2DateTimeMax(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 23:59:59"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return parse(timeStr + lastStr); + } + + return null; + } + + /** + * 字符串数字转化为double + * + * @param string + * @return + */ + public static double str2double(String string) { + if (null == string || "".equals(string)) { + return 0; + } + try { + return Double.valueOf(string); + } catch (NumberFormatException e) { + return 0; + } + } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/HttpClientUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/HttpClientUtil.java new file mode 100644 index 000000000..fc3131629 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/HttpClientUtil.java @@ -0,0 +1,444 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.util; + +import org.apache.http.Consts; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.SocketTimeoutException; +import java.net.URLEncoder; +import java.security.cert.CertificateException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.Map.Entry; + +public final class HttpClientUtil { + private final static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); + public final static int connectTimeout = 5000; + private static PoolingHttpClientConnectionManager connManager = null; + private static CloseableHttpClient httpclient = null; + + private static TrustManager trustAllManager = new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + }; + + static { + httpclient = HttpClients.createDefault(); + } + + public static String postForm(String url, int timeout, Map headerMap, List paramsList, String encoding){ + HttpPost post = new HttpPost(url); + try { + if(headerMap != null){ + for(Entry entry : headerMap.entrySet()){ + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + //post.setHeader("Content-type", "application/json"); + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(timeout) + .setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + + post.setEntity(new UrlEncodedFormEntity(paramsList, encoding)); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if(entity != null){ + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if(entity != null){ + entity.getContent().close(); + } + } + } finally { + if(response != null){ + response.close(); + } + } + } catch (Exception e) { + throw new RuntimeException("invoke http post error!",e); + } finally { + post.releaseConnection(); + } + return ""; + } + + public static String postJsonBody(String url, int timeout, Map headerMap, + String paraData, String encoding) { + + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + StringEntity jsonEntity = new StringEntity(paraData, ContentType.APPLICATION_JSON); + post.setConfig(requestConfig); + post.setEntity(jsonEntity); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + @SuppressWarnings("deprecation") + public static String invokeGet(String url, Map params, String encode, int connectTimeout, + int soTimeout) { + String responseString = null; + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + + StringBuilder sb = new StringBuilder(); + sb.append(url); + int i = 0; + if (params != null) { + for (Entry entry : params.entrySet()) { + if (i == 0 && !url.contains("?")) { + sb.append("?"); + } else { + sb.append("&"); + } + sb.append(entry.getKey()); + sb.append("="); + String value = entry.getValue(); + try { + sb.append(URLEncoder.encode(value, "UTF-8")); + } catch (UnsupportedEncodingException e) { + logger.warn("encode http get params error, value is " + value, e); + sb.append(URLEncoder.encode(value)); + } + i++; + } + } + HttpGet get = new HttpGet(sb.toString()); + get.setConfig(requestConfig); + try { + CloseableHttpResponse response = httpclient.execute(get); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + responseString = EntityUtils.toString(entity, encode); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]get response error, url:%s", sb.toString()), e); + return responseString; + } finally { + if (response != null) { + response.close(); + } + } + // System.out.println(String.format("[HttpUtils Get]Debug url:%s , + // response string %s:", sb.toString(), responseString)); + } catch (SocketTimeoutException e) { + logger.error(String.format("[HttpUtils Get]invoke get timout error, url:%s", sb.toString()), e); + return responseString; + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]invoke get error, url:%s", sb.toString()), e); + } finally { + get.releaseConnection(); + } + return responseString; + } + + /** + * HTTPS请求,默认超时为5S + * + * @param reqURL + * @param params + * @return + */ + public static String connectPostHttps(String reqURL, Map params) { + + String responseContent = null; + HttpPost httpPost = new HttpPost(reqURL); + try { + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + List formParams = new ArrayList(); + httpPost.setEntity(new UrlEncodedFormEntity(formParams, Consts.UTF_8)); + httpPost.setConfig(requestConfig); + // 绑定到请求 Entry + for (Entry entry : params.entrySet()) { + formParams.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); + } + CloseableHttpResponse response = httpclient.execute(httpPost); + try { + // 执行POST请求 + HttpEntity entity = response.getEntity(); // 获取响应实体 + try { + if (null != entity) { + responseContent = EntityUtils.toString(entity, Consts.UTF_8); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + logger.info("requestURI : " + httpPost.getURI() + ", responseContent: " + responseContent); + } catch (ClientProtocolException e) { + logger.error("ClientProtocolException", e); + } catch (IOException e) { + logger.error("IOException", e); + } finally { + httpPost.releaseConnection(); + } + return responseContent; + + } + + class Test { + String v; + String k; + + public String getV() { + return v; + } + + public void setV(String v) { + this.v = v; + } + + public String getK() { + return k; + } + + public void setK(String k) { + this.k = k; + } + + } + + // 随机4位数 + public static String getRandomValue() { + String str = "0123456789"; + StringBuilder sb = new StringBuilder(4); + for (int i = 0; i < 4; i++) { + char ch = str.charAt(new Random().nextInt(str.length())); + sb.append(ch); + } + return sb.toString(); + } + + // 当前时间到秒 + public static String getTimestamp() { + + Date date = new Date(); + String timestamp = String.valueOf(date.getTime() / 1000); + return timestamp; + } + + // 当前时间到秒 + public static String getNowDate() { + Date date = new Date(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); + return sdf.format(date); + } + + public static String postJsonBody2(String url, int timeout, Map headerMap, + List paramsList, String encoding) { + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(paramsList, encoding); + post.setEntity(entity); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + public static String postJsonBody3(String url, int timeout, Map headerMap, + Map paramsList, String encoding) { + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + //JSONArray jsonArray = JSONArray.fromObject(paramsList); + //post.setEntity(new StringEntity(jsonArray.get(0).toString(), encoding)); + post.setEntity(new StringEntity(null, encoding)); + //logger.info("successfully start post Json Body url{},params ", url,jsonArray.get(0).toString()); + logger.info("successfully start post Json Body url{},params ", url,null); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + public static String executeGet(String url) + { + String rtnStr = ""; + HttpGet httpGet = new HttpGet(url); + try { + HttpResponse httpResponse = httpclient.execute(httpGet); + //获得返回的结果 + rtnStr = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + e.printStackTrace(); + } finally { + httpGet.releaseConnection(); + } + return rtnStr; + } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ModelMapperUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ModelMapperUtil.java new file mode 100644 index 000000000..89b49d23d --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ModelMapperUtil.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.util; + +import org.modelmapper.ModelMapper; +import org.modelmapper.convention.MatchingStrategies; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + + +public class ModelMapperUtil { + private static ModelMapper modelMapper = new ModelMapper(); + + public ModelMapperUtil() { + } + + public static D strictMap(Object source, Class destinationType) { + return modelMapper.map(source, destinationType); + } + + public static List strictMapList(Object source, Class componentType) { + List list = new ArrayList(); + List objectList = (List)source; + Iterator var4 = objectList.iterator(); + + while(var4.hasNext()) { + Object obj = var4.next(); + list.add(modelMapper.map(obj, componentType)); + } + + return list; + } + + static { + modelMapper.getConfiguration().setMatchingStrategy(MatchingStrategies.STRICT); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/SQLCheckUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/SQLCheckUtil.java new file mode 100644 index 000000000..54b71be63 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/SQLCheckUtil.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class SQLCheckUtil { + + // sql 注入检查 + private static String injectionReg = "(?:--)|" + + "(\\b(select|update|union|and|or|delete|insert|trancate|char|substr|ascii|declare|exec|count|master|into|drop|execute)\\b)"; + private static Pattern paramInjectionPattern = Pattern.compile(injectionReg, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + /** + * 1. -- 注释 + * 2. select * from table -- 注释 + * 但是不包含 -- '注释' + * */ + private static String sqlCommentReg = "\\-\\-([^\\'\\r\\n]{0,}(\\'[^\\'\\r\\n]{0,}\\'){0,1}[^\\'\\r\\n]{0,}){0,}"; + private static Pattern sqlCommentPattern = Pattern.compile(sqlCommentReg, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + public static boolean doParamInjectionCheck(String str) { + Matcher matcher = paramInjectionPattern.matcher(str); + if(matcher.find()) { + return true; + } + return false; + } + + public static String sqlCommentReplace(String sql) { + String newSql = sql.replaceAll(sqlCommentReg, ""); + return newSql; + } + + public static void main(String[] args) { + String sql = "--注释\n" + + "-- 注释\n" + + "select * from tb --注释\n"; + String newSql = sqlCommentReplace(sql); + String[] selects = newSql.split("select"); + System.out.println(newSql); + System.out.println(selects.length); + } + +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/UUIDGenerator.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/UUIDGenerator.java new file mode 100644 index 000000000..5c0adc70b --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/UUIDGenerator.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.util; + +import java.util.UUID; + + +public class UUIDGenerator { + + /** + * 生成唯一标识uuid + * */ + public static String genUUID() { + String uuid = UUID.randomUUID().toString(); + return uuid; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiAccessVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiAccessVo.java new file mode 100644 index 000000000..d75bac971 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiAccessVo.java @@ -0,0 +1,100 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.vo; + + +public class ApiAccessVo { + Long id; + + String user; + + String apiServiceName; + + Long apiServiceId; + + Long apiServiceVersionId; + + String apiPublisher; + + String accessTime; + + String proxyUser; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getApiServiceName() { + return apiServiceName; + } + + public void setApiServiceName(String apiServiceName) { + this.apiServiceName = apiServiceName; + } + + public Long getApiServiceId() { + return apiServiceId; + } + + public void setApiServiceId(Long apiServiceId) { + this.apiServiceId = apiServiceId; + } + + public Long getApiServiceVersionId() { + return apiServiceVersionId; + } + + public void setApiServiceVersionId(Long apiServiceVersionId) { + this.apiServiceVersionId = apiServiceVersionId; + } + + public String getApiPublisher() { + return apiPublisher; + } + + public void setApiPublisher(String apiPublisher) { + this.apiPublisher = apiPublisher; + } + + public String getAccessTime() { + return accessTime; + } + + public void setAccessTime(String accessTime) { + this.accessTime = accessTime; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiServiceVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiServiceVo.java new file mode 100644 index 000000000..ff4858499 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiServiceVo.java @@ -0,0 +1,279 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + + +import java.util.Date; +import java.util.List; +import java.util.Map; + + +public class ApiServiceVo { + private Long id; + private String name; + private String aliasName; + private String path; + private int protocol; + private String method; + private String tag; + private String scope; + private String description; + private Integer status = 1; + private String type; + private String runType; + private Date createTime; + private Date modifyTime; + private String creator; + private String modifier; + private String scriptPath; + private ApprovalVo approvalVo; + + private Long latestVersionId; + private String userToken; + + private List params; + + private Map metadata; + private String content; + + private List versionVos; + + private Long workspaceId = 1L; + private Long targetServiceId; + private String comment; + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + + public Long getTargetServiceId() { + return targetServiceId; + } + + public void setTargetServiceId(Long targetServiceId) { + this.targetServiceId = targetServiceId; + } + + + + public ApprovalVo getApprovalVo() { + return approvalVo; + } + + public void setApprovalVo(ApprovalVo approvalVo) { + this.approvalVo = approvalVo; + } + + public Long getLatestVersionId() { + return latestVersionId; + } + + public void setLatestVersionId(Long latestVersionId) { + this.latestVersionId = latestVersionId; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public int getProtocol() { + return protocol; + } + + public void setProtocol(int protocol) { + this.protocol = protocol; + } + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getModifier() { + return modifier; + } + + public void setModifier(String modifier) { + this.modifier = modifier; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public String getScriptPath() { + return scriptPath; + } + + public void setScriptPath(String scriptPath) { + this.scriptPath = scriptPath; + } + + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public List getVersionVos() { + return versionVos; + } + + public void setVersionVos(List versionVos) { + this.versionVos = versionVos; + } + + public List getParams() { + return params; + } + + public void setParams(List params) { + this.params = params; + } + + + public String getUserToken() { + return userToken; + } + + public void setUserToken(String userToken) { + this.userToken = userToken; + } + public String getRunType() { + return runType; + } + + public void setRunType(String runType) { + this.runType = runType; + } + + public String getAliasName() { + return aliasName; + } + + public void setAliasName(String aliasName) { + this.aliasName = aliasName; + } + +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiVersionVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiVersionVo.java new file mode 100644 index 000000000..cfd82fc86 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiVersionVo.java @@ -0,0 +1,140 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + +import java.util.Date; +import java.util.List; + + +public class ApiVersionVo { + + private Long id; + private Long apiId; + private String version; + private String bmlResourceId; + private String bmlVersion; + private String source; + private String creator; + private Date createTime; + private String metadataInfo; + private String authId; + + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + private int status; + + /** + * one version data corresponds to multiple param data + * */ + private List paramVos; + + public List getParamVos() { + return paramVos; + } + + public String getMetadataInfo() { + return metadataInfo; + } + + public String getAuthId() { + return authId; + } + + public void setAuthId(String authId) { + this.authId = authId; + } + + public void setMetadataInfo(String metadataInfo) { + this.metadataInfo = metadataInfo; + } + + public void setParamVos(List paramVos) { + this.paramVos = paramVos; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getApiId() { + return apiId; + } + + public void setApiId(Long apiId) { + this.apiId = apiId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getBmlResourceId() { + return bmlResourceId; + } + + public void setBmlResourceId(String bmlResourceId) { + this.bmlResourceId = bmlResourceId; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApprovalVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApprovalVo.java new file mode 100644 index 000000000..2b1eeec1b --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApprovalVo.java @@ -0,0 +1,122 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.vo; + +import java.util.Date; + + +public class ApprovalVo { + Long id; + Long apiId; + Long apiVersionId; + String approvalName; + String applyUser; + String executeUser; + String creator; + Integer status; + Date createTime; + Date updateTime; + String approvalNo; + + public String getCreator() { + return creator; + } + + public Long getApiVersionId() { + return apiVersionId; + } + + public void setApiVersionId(Long apiVersionId) { + this.apiVersionId = apiVersionId; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getApiId() { + return apiId; + } + + public void setApiId(Long apiId) { + this.apiId = apiId; + } + + public String getApprovalName() { + return approvalName; + } + + public void setApprovalName(String approvalName) { + this.approvalName = approvalName; + } + + public String getApplyUser() { + return applyUser; + } + + public void setApplyUser(String applyUser) { + this.applyUser = applyUser; + } + + public String getExecuteUser() { + return executeUser; + } + + public void setExecuteUser(String executeUser) { + this.executeUser = executeUser; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getApprovalNo() { + return approvalNo; + } + + public void setApprovalNo(String approvalNo) { + this.approvalNo = approvalNo; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageExtVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageExtVo.java new file mode 100644 index 000000000..84790e17f --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageExtVo.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + +import java.util.LinkedHashMap; + + +public class MessageExtVo extends LinkedHashMap { + public static MessageExtVo build(MessageVo messageVo) { + MessageExtVo messageExtVo = new MessageExtVo(); + + messageExtVo.put("status", messageVo.getStatus()); + messageExtVo.put("message", messageVo.getMessage()); + messageExtVo.put("data", messageVo.getData()); + + return messageExtVo; + } + + public MessageExtVo setMessage(String message) { + put("message", message); + return this; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageVo.java new file mode 100644 index 000000000..39e1dddc8 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageVo.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + + +public class MessageVo { + private Integer status = 0; + + private String message = "OK"; + + private Object data; + + public MessageVo() { + } + + public MessageVo(Integer status, String message, Object data) { + this.status = status; + this.message = message; + this.data = data; + } + + public Integer getStatus() { + return status; + } + + public MessageVo setStatus(Integer status) { + this.status = status; + return this; + } + + public String getMessage() { + return message; + } + + public MessageVo setMessage(String message) { + this.message = message; + return this; + } + + public Object getData() { + return data; + } + + public MessageVo setData(Object data) { + this.data = data; + return this; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ParamVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ParamVo.java new file mode 100644 index 000000000..2dc6ccffd --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ParamVo.java @@ -0,0 +1,102 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + + +public class ParamVo { + + private Long id; + private Long apiVersionId; + private String name; + private String type; + private Integer required = 1; + private String description; + private String displayName; + private String defaultValue; + private String details; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getApiVersionId() { + return apiVersionId; + } + + public void setApiVersionId(Long apiVersionId) { + this.apiVersionId = apiVersionId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Integer getRequired() { + return required; + } + + public void setRequired(Integer required) { + this.required = required; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public String getDetails() { + return details; + } + + public void setDetails(String details) { + this.details = details; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/QueryParamVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/QueryParamVo.java new file mode 100644 index 000000000..cfc184eae --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/QueryParamVo.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + + +public class QueryParamVo extends ParamVo { + + private String typeStr; + + private String requireStr; + + private String testValue; + + public String getTypeStr() { + return typeStr; + } + + public void setTypeStr(String typeStr) { + this.typeStr = typeStr; + } + + public String getRequireStr() { + return requireStr; + } + + public void setRequireStr(String requireStr) { + this.requireStr = requireStr; + } + + public String getTestValue() { + return testValue; + } + + public void setTestValue(String testValue) { + this.testValue = testValue; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/TokenManagerVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/TokenManagerVo.java new file mode 100644 index 000000000..13f239b5f --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/TokenManagerVo.java @@ -0,0 +1,148 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.vo; + +import java.util.Date; + + +public class TokenManagerVo { + private Long id; + private Long apiId; + private Long apiVersionId; + public String publisher; + private String user; + private Date applyTime; + private Long duration; + private String reason; + private String ipWhitelist; + private Integer status; + private String caller; + private String accessLimit; + private String token; + private String applySource; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getApiId() { + return apiId; + } + + public void setApiId(Long apiId) { + this.apiId = apiId; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPublisher() { + return publisher; + } + + public void setPublisher(String publisher) { + this.publisher = publisher; + } + + public Date getApplyTime() { + return applyTime; + } + + public void setApplyTime(Date applyTime) { + this.applyTime = applyTime; + } + + public Long getDuration() { + return duration; + } + + public void setDuration(Long duration) { + this.duration = duration; + } + + public String getReason() { + return reason; + } + + public void setReason(String reason) { + this.reason = reason; + } + + public String getIpWhitelist() { + return ipWhitelist; + } + + public void setIpWhitelist(String ipWhitelist) { + this.ipWhitelist = ipWhitelist; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getCaller() { + return caller; + } + + public void setCaller(String caller) { + this.caller = caller; + } + + public String getAccessLimit() { + return accessLimit; + } + + public void setAccessLimit(String accessLimit) { + this.accessLimit = accessLimit; + } + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } + + public String getApplySource() { + return applySource; + } + + public void setApplySource(String applySource) { + this.applySource = applySource; + } + public Long getApiVersionId() { + return apiVersionId; + } + + public void setApiVersionId(Long apiVersionId) { + this.apiVersionId = apiVersionId; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/scala/com/webank/wedatasphere/dss/apiservice/DSSApiServiceServerApplication.scala b/dss-apps/dss-apiservice-server/src/main/scala/com/webank/wedatasphere/dss/apiservice/DSSApiServiceServerApplication.scala new file mode 100644 index 000000000..747a34fcd --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/scala/com/webank/wedatasphere/dss/apiservice/DSSApiServiceServerApplication.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + + +object DSSApiServiceServerApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/test/java/com/webank/wedatasphere/dss/apiservice/test/TestApiServiceDBOperation.java b/dss-apps/dss-apiservice-server/src/test/java/com/webank/wedatasphere/dss/apiservice/test/TestApiServiceDBOperation.java new file mode 100644 index 000000000..c096377c5 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/test/java/com/webank/wedatasphere/dss/apiservice/test/TestApiServiceDBOperation.java @@ -0,0 +1,157 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.test; + +import com.ninja_squad.dbsetup.DbSetup; +import com.ninja_squad.dbsetup.DbSetupTracker; +import com.ninja_squad.dbsetup.Operations; +import com.ninja_squad.dbsetup.destination.DataSourceDestination; +import com.ninja_squad.dbsetup.operation.Operation; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceAccessDao; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceDao; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiService; +import com.webank.wedatasphere.dss.apiservice.core.token.JwtManager; +import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiAccessVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiServiceVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiVersionVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import org.apache.linkis.DataWorkCloudApplication; +import org.junit.Before; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.sql.DataSource; +import java.util.Date; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + + + +//@RunWith(SpringJUnit4ClassRunner.class) +//@MapperScan(annotationClass = Repository.class, basePackages = "com.webank.wedatasphere.dss.apiservice.dao" ) +//@SpringBootTest(classes = {DataWorkCloudApplication.class}) +public class TestApiServiceDBOperation { + /*private static DbSetupTracker dbSetupTracker = new DbSetupTracker(); + @Autowired + ApiService apiService; + @Autowired + DataSource dataSource; + @Autowired + ApiServiceDao apiServiceDao; + + + @Autowired + ApiServiceAccessDao apiServiceAccessDao; + + @Before + public void setUp() { + Operation operation = Operations.sequenceOf( + Operations.deleteAllFrom("dss_apiservice_api", "dss_apiservice_param"), + Operations.insertInto("dss_apiservice_api") + .columns("name", "alias_name", "path", "protocol", "method", "tag", "scope", "description", "status", "type", "run_type", "create_time", "modify_time", "creator", "modifier", "script_path", "workspaceID", "api_comment") + .values("test01", "测试", "/test01", 0, "get", "new test", "shouquankejian", "miaoshu", "1", "sql", "spark", "2020-10-19 11:00:00", "2020-11-19 12:00:00", "allenlliu", "allenlliu", "123.sql", "180", "comment test") + .build(), + Operations.insertInto("dss_apiservice_param") + .columns("api_version_id", "name", "display_name", "type", "required", "default_value", "description", "details") + .values(1, "idcard", "身份证", "String", 1, "430124000000000", "test param", "身份标识") + .build() +// Operations.insertInto("dss_apiservice_access_info") +// .columns("api_version_id", "login_user", "display_name", "type", "required", "default_value", "description", "details").build(), + ); + DbSetup dbSetup = new DbSetup(new DataSourceDestination(dataSource), operation); + dbSetupTracker.launchIfNecessary(dbSetup); + } + + @DisplayName("ApiService库表验证") + @Test + public void testApiServiceDBOperate() { + dbSetupTracker.skipNextLaunch(); + + Integer actual = apiServiceDao.enableApi(0L); + ApiServiceVo apiServiceVo = apiServiceDao.queryByPath("/test01"); + Integer count = apiService.queryCountByName("test01"); + + Assertions.assertEquals(apiServiceVo.getName(), "test01"); + assertThat(actual, equalTo(0)); + assertThat(count, equalTo(1)); + } + + @DisplayName("ApiService备注更新测试") + @Test + public void testApiServiceCommentUpdate() { + dbSetupTracker.skipNextLaunch(); + + ApiServiceVo apiServiceVo01 = apiServiceDao.queryByPath("/test01"); + Integer actual = apiServiceDao.updateApiServiceComment(apiServiceVo01.getId(), "new comment"); + ApiServiceVo apiServiceVo02 = apiServiceDao.queryByPath("/test01"); + + + Assertions.assertEquals(apiServiceVo02.getComment(), "new comment"); + } + + @DisplayName("ApiService保存和版本验证") + @Test + public void testApiServiceVersion() { + dbSetupTracker.skipNextLaunch(); + + + ApiServiceVo apiServiceVo = apiServiceDao.queryByPath("/test01"); + apiServiceVo.setName("test02"); + apiServiceVo.setPath("/test02"); + //BML会上传失败,没有配置实际环境 + Assertions.assertThrows(NullPointerException.class, () -> { + apiService.save(apiServiceVo); + ; + }); + + } + + + + @DisplayName("数据服务访问记录验证") + @Test + public void testApiServiceAccessInfo() { + System.out.println("数据服务访问记录验证"); + ApiAccessVo apiAccessVo = new ApiAccessVo(); + apiAccessVo.setUser("allenlliu"); + apiAccessVo.setApiPublisher("testUser"); + apiAccessVo.setApiServiceName("testApi"); + apiAccessVo.setApiServiceId(102L); + apiAccessVo.setApiServiceVersionId(10L); + apiAccessVo.setProxyUser("hadoop"); + apiAccessVo.setAccessTime(DateUtil.getNow()); + apiServiceAccessDao.addAccessRecord(apiAccessVo); + + ApiAccessVo targetAccessVo = apiServiceAccessDao.queryByVersionId(10L); + Assertions.assertAll("accessInfos", + () -> Assertions.assertEquals(targetAccessVo.getProxyUser(), "hadoop"), + () -> Assertions.assertEquals(targetAccessVo.getApiServiceName(), "testApi"), + () -> Assertions.assertEquals(targetAccessVo.getApiPublisher(), "testUser") + + ); + + }*/ +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties b/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties new file mode 100644 index 000000000..69cd5c3fe --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties @@ -0,0 +1,57 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# mysql 驱动: h2 +spring.datasource.driver-class-name=org.h2.Driver +# h2 内存数据库 库名: test +spring.datasource.url=jdbc:h2:mem:test +# 初始化数据表 +spring.datasource.schema=classpath:schema.sql +spring.datasource.username= +spring.datasource.password= +# 打印 SQL语句, Mapper所处的包 +logging.level.com.hawkingfoo.dao=debug + + + +wds.linkis.server.mybatis.mapperLocations=classpath:com/webank/wedatasphere/dss/apiservice/core/dao/mapper/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.apiservice.core +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.apiservice.core.dao + +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.apiservice.core.restful + +#sit +wds.linkis.server.version=v1 +wds.linkis.server.url= + +#test +wds.linkis.test.mode=false +wds.linkis.test.user= +wds.linkis.server.mybatis.datasource.url=jdbc:h2:mem:test +wds.linkis.server.mybatis.datasource.username= +wds.linkis.server.mybatis.datasource.password= + +#dsm +wds.linkis.server.dsm.admin.users= + +#bml +wds.linkis.gateway.url= +wds.linkis.gateway.ip= +wds.linkis.gateway.port= + +#用于执行的datasource配置 +wds.linkis.datasource.hikari.maximumPoolSize=100 +wds.linkis.datasource.hikari.minimumIdle=10 diff --git a/dss-apps/dss-apiservice-server/src/test/resources/schema.sql b/dss-apps/dss-apiservice-server/src/test/resources/schema.sql new file mode 100644 index 000000000..905024b93 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/test/resources/schema.sql @@ -0,0 +1,66 @@ +CREATE TABLE dss_apiservice_api ( + id bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + name varchar(200) NOT NULL COMMENT '服务名称', + alias_name varchar(200) NOT NULL COMMENT '服务中文名称', + path varchar(200) NOT NULL COMMENT '服务路径', + protocol int(11) NOT NULL COMMENT '协议: http, https', + method varchar(10) NOT NULL COMMENT '方法: post, put, delete', + tag varchar(200) DEFAULT NULL COMMENT '标签', + scope varchar(50) DEFAULT NULL COMMENT '范围', + description varchar(200) DEFAULT NULL COMMENT '服务描述', + status int(11) DEFAULT '0' COMMENT '服务状态,默认0是停止,1是运行中', + type varchar(50) DEFAULT NULL COMMENT '服务引擎类型', + run_type varchar(50) DEFAULT NULL COMMENT '脚本类型', + create_time timestamp NOT NULL COMMENT '创建时间', + modify_time timestamp NOT NULL COMMENT '修改时间', + creator varchar(50) DEFAULT NULL COMMENT '创建者', + modifier varchar(50) DEFAULT NULL COMMENT '修改者', + script_path varchar(200) NOT NULL COMMENT '脚本路径', + workspaceID int(11) NOT NULL COMMENT '工作空间ID', + api_comment varchar(1024) DEFAULT NULL COMMENT '服务备注', + PRIMARY KEY ( id ) +) COMMENT='服务api配置表'; + + +CREATE TABLE dss_apiservice_param ( + id bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + api_version_id bigint(20) NOT NULL COMMENT '服务api版本id', + name varchar(200) NOT NULL COMMENT '名称', + display_name varchar(50) DEFAULT NULL COMMENT '展示名', + type varchar(50) DEFAULT NULL COMMENT '类型', + required tinyint(1) DEFAULT '1' COMMENT '是否必须: 0否, 1是', + default_value varchar(200) DEFAULT NULL COMMENT '参数的默认值', + description varchar(200) DEFAULT NULL COMMENT '描述', + details varchar(500) DEFAULT NULL COMMENT '变量的详细说明', + PRIMARY KEY ( id ) +) COMMENT='apiservice 参数表'; + + +CREATE TABLE `dss_apiservice_api_version` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_id` bigint(20) NOT NULL COMMENT '服务的ID', + `version` varchar(50) NOT NULL COMMENT '服务对应的版本信息', + `bml_resource_id` varchar(50) NOT NULL COMMENT 'bml资源id', + `bml_version` varchar(20) NOT NULL COMMENT 'bml版本', + `source` varchar(200) DEFAULT NULL COMMENT '来源', + `creator` varchar(50) DEFAULT NULL COMMENT '创建者', + `create_time`timestamp NOT NULL COMMENT '创建时间', + `status` tinyint(1) default '1' COMMENT '0表示被禁用,1表示正在运行', + `metadata_info` varchar(5000) NOT NULL COMMENT '发布者库表信息', + `auth_id` varchar(200) NOT NULL COMMENT '用于与datamap交互的UUID', + `datamap_order_no` varchar(200) DEFAULT NULL COMMENT 'datamap审批单号码', + PRIMARY KEY(`id`) +) COMMENT='服务api版本表'; + + +CREATE TABLE `dss_apiservice_access_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_id` bigint(20) NOT NULL COMMENT '服务id', + `api_version_id` bigint(20) NOT NULL COMMENT '版本id', + `api_name` varchar(50) NOT NULL COMMENT '服务名称', + `login_user` varchar(50) NOT NULL COMMENT '提交用户', + `execute_user` varchar(50) DEFAULT NULL COMMENT '代理执行用户', + `api_publisher` varchar(50) NOT NULL COMMENT 'api创建者', + `access_time` timestamp NOT null COMMENT '访问时间', + PRIMARY KEY(`id`) +) COMMENT='apiservice 访问信息表'; \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-api-sql-template/pom.xml b/dss-apps/dss-data-api/dss-api-sql-template/pom.xml new file mode 100644 index 000000000..854c15cb3 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/pom.xml @@ -0,0 +1,52 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-api-sql-template + + + + dom4j + dom4j + 1.6.1 + + + ognl + ognl + 2.7.3 + + + + org.apache.commons + commons-lang3 + 3.8.1 + provided + + + + + + + + + + junit + junit + 4.13.1 + test + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/Cache.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/Cache.java new file mode 100644 index 000000000..f36c10662 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/Cache.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.orange; + +import com.webank.wedatasphere.dss.orange.node.SqlNode; + +import java.util.concurrent.ConcurrentHashMap; + + +public class Cache { + + ConcurrentHashMap nodeCache = new ConcurrentHashMap<>(); + + public ConcurrentHashMap getNodeCache() { + return nodeCache; + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/DynamicSqlEngine.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/DynamicSqlEngine.java new file mode 100644 index 000000000..9de2e3c9d --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/DynamicSqlEngine.java @@ -0,0 +1,85 @@ +package com.webank.wedatasphere.dss.orange; + +import com.webank.wedatasphere.dss.orange.context.Context; +import com.webank.wedatasphere.dss.orange.node.SqlNode; +import com.webank.wedatasphere.dss.orange.tag.XmlParser; +import com.webank.wedatasphere.dss.orange.token.TokenHandler; +import com.webank.wedatasphere.dss.orange.token.TokenParser; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + + +public class DynamicSqlEngine { + + Cache cache = new Cache(); + + public SqlMeta parse(String text, Map params) { + text = String.format("%s", text); + SqlNode sqlNode = parseXml2SqlNode(text); + Context context = new Context(params); + parseSqlText(sqlNode, context); + parseParameter(context); + SqlMeta sqlMeta = new SqlMeta(context.getSql(), context.getJdbcParameters()); + return sqlMeta; + } + + public Set parseParameter(String text) { + text = String.format("%s", text); + SqlNode sqlNode = parseXml2SqlNode(text); + HashSet set = new HashSet<>(); + sqlNode.applyParameter(set); + return set; + } + + private SqlNode parseXml2SqlNode(String text) { + SqlNode node = cache.getNodeCache().get(text); + if (node == null) { + node = XmlParser.parseXml2SqlNode(text); + cache.getNodeCache().put(text, node); + } + return node; + } + + /** + * 解析标签,去除标签,替换 ${}为常量值, #{}保留不变 + * + * @param sqlNode + * @param context + */ + private void parseSqlText(SqlNode sqlNode, Context context) { + sqlNode.apply(context); + } + + /** + * #{}替换成?,并且将?对应的参数值按顺序保存起来 + * + * @param context + */ + private void parseParameter(Context context) { + TokenParser tokenParser = new TokenParser("#{", "}", new TokenHandler() { + @Override + public String handleToken(String content) { + Object value = context.getOgnlValue(content); + if (value == null) { + throw new RuntimeException("could not found value : " + content); + } + context.addParameter(value); + return "?"; + } + }); + String sql = tokenParser.parse(context.getSql()); + context.setSql(sql); + } + + public static void main(String[] args) { + DynamicSqlEngine engine = new DynamicSqlEngine(); + String sql = ("select id > ${minId} #{minId} and id < ${maxId} #{maxId} "); + Map map = new HashMap<>(); + map.put("minId", 100); + map.put("maxId", 500); + engine.parse(sql, map); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/SqlMeta.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/SqlMeta.java new file mode 100644 index 000000000..d86f472b6 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/SqlMeta.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.orange; + +import java.util.List; + + +public class SqlMeta { + + String sql; + List jdbcParamValues; + + public SqlMeta(String sql, List jdbcParamValues) { + this.sql = sql; + this.jdbcParamValues = jdbcParamValues; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public List getJdbcParamValues() { + return jdbcParamValues; + } + + public void setJdbcParamValues(List jdbcParamValues) { + this.jdbcParamValues = jdbcParamValues; + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/context/Context.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/context/Context.java new file mode 100644 index 000000000..1aa4a92c0 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/context/Context.java @@ -0,0 +1,64 @@ +package com.webank.wedatasphere.dss.orange.context; + +import com.webank.wedatasphere.dss.orange.util.OgnlUtil; + +import java.util.*; + + +public class Context { + + StringBuilder sqlBuilder = new StringBuilder(); + List jdbcParameters = new ArrayList<>(); + Set paramNames = new HashSet<>(); + + // List jdbcParameterNames = new ArrayList<>(); + Map data; + + public Context(Map data) { + this.data = data; + } + + public void appendSql(String text) { + if (text != null) + sqlBuilder.append(text); + } + + public void addParameter(Object o) { + jdbcParameters.add(o); + } + + public void addParameterName(String o) { + paramNames.add(o); + } + + /** + * 通过ognl表达式获取值 + * + * @param expression + * @return + */ + public Object getOgnlValue(String expression) { + return OgnlUtil.getValue(expression, data); + } + + public Boolean getOgnlBooleanValue(String expression) { + return OgnlUtil.getBooleanValue(expression, data); + } + + public String getSql() { + return sqlBuilder.toString(); + } + + public void setSql(String text) { + sqlBuilder = new StringBuilder(text); + } + + public List getJdbcParameters() { + return jdbcParameters; + } + + public Map getData() { + return data; + } + +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/ForeachSqlNode.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/ForeachSqlNode.java new file mode 100644 index 000000000..8631bcf81 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/ForeachSqlNode.java @@ -0,0 +1,103 @@ +package com.webank.wedatasphere.dss.orange.node; + +import com.webank.wedatasphere.dss.orange.context.Context; +import com.webank.wedatasphere.dss.orange.token.TokenHandler; +import com.webank.wedatasphere.dss.orange.token.TokenParser; +import com.webank.wedatasphere.dss.orange.util.OgnlUtil; +import com.webank.wedatasphere.dss.orange.util.RegexUtil; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; + + +public class ForeachSqlNode implements SqlNode { + + String collection; + String open; + String close; + String separator; + String item; + String index; + SqlNode contents; + + String indexDataName; + + public ForeachSqlNode(String collection, String open, String close, String separator, String item, String index, SqlNode contents) { + this.collection = collection; + this.open = open; + this.close = close; + this.separator = separator; + this.item = item; + this.index = index; + this.contents = contents; + + this.indexDataName = String.format("__index_%s", collection); + } + + @Override + public void apply(Context context) { + context.appendSql(" ");//标签类SqlNode先拼接空格,和前面的内容隔开 + Iterable iterable = OgnlUtil.getIterable(collection, context.getData()); + int currentIndex = 0; + + ArrayList indexs = new ArrayList<>(); + + context.getData().put(indexDataName, indexs); + context.appendSql(open); + + for (Object o : iterable) { + + ((ArrayList) context.getData().get(indexDataName)).add(currentIndex); + //不是第一次,需要拼接分隔符 + if (currentIndex != 0) { + context.appendSql(separator); + } + + Context proxy = new Context(context.getData()); + String childSqlText = getChildText(proxy, currentIndex); + context.appendSql(childSqlText); + + currentIndex++; + } + + context.appendSql(close); + + } + + @Override + public void applyParameter(Set set) { + set.add(collection); + Set temp = new HashSet<>(); + contents.applyParameter(set); + for (String key: temp){ + if (key.matches(item + "[.,:\\s\\[]")) + continue; + if (key.matches(index + "[.,:\\s\\[]")) + continue; + set.add(key); + } + } + + public String getChildText(Context proxy, int currentIndex) { + String newItem = String.format("%s[%d]", collection, currentIndex); //ognl可以直接获取 aaa[0] 形式的值 + String newIndex = String.format("%s[%d]", indexDataName, currentIndex); + this.contents.apply(proxy); + String sql = proxy.getSql(); + TokenParser tokenParser = new TokenParser("#{", "}", new TokenHandler() { + @Override + public String handleToken(String content) { + //item替换成自己的变量名: item[0] item[1] item[2] ...... + String replace = RegexUtil.replace(content, item, newItem); + if (replace.equals(content)) + //index替换成自己的变量名: __index_xxx[0] __index_xxx[1] __index_xxx[2] ...... + replace = RegexUtil.replace(content, index, newIndex); + StringBuilder builder = new StringBuilder(); + return builder.append("#{").append(replace).append("}").toString(); + } + }); + String parse = tokenParser.parse(sql); + return parse; + } + +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/IfSqlNode.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/IfSqlNode.java new file mode 100644 index 000000000..d141c358d --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/IfSqlNode.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.orange.node; + +import com.webank.wedatasphere.dss.orange.context.Context; + +import java.util.Set; + + +public class IfSqlNode implements SqlNode { + + String test; + + SqlNode contents; + + public IfSqlNode(String test, SqlNode contents) { + this.test = test; + this.contents = contents; + } + + @Override + public void apply(Context context) { + Boolean value = context.getOgnlBooleanValue(test); + if (value) { + context.appendSql(" ");//标签类SqlNode先拼接空格,和前面的内容隔开 + contents.apply(context); + } + + } + + @Override + public void applyParameter(Set set) { + contents.applyParameter(set); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/MixedSqlNode.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/MixedSqlNode.java new file mode 100644 index 000000000..c2357b77a --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/MixedSqlNode.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.orange.node; + +import com.webank.wedatasphere.dss.orange.context.Context; + +import java.util.List; +import java.util.Set; + + +public class MixedSqlNode implements SqlNode { + + List contents ; + + public MixedSqlNode(List contents) { + this.contents = contents; + } + + @Override + public void apply(Context context) { + for (SqlNode node: contents){ + node.apply(context); + } + } + + @Override + public void applyParameter(Set set) { + for (SqlNode node: contents){ + node.applyParameter(set); + } + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/SetSqlNode.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/SetSqlNode.java new file mode 100644 index 000000000..00e907562 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/SetSqlNode.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.orange.node; + +import java.util.Arrays; + + +public class SetSqlNode extends TrimSqlNode { + + public SetSqlNode(SqlNode contents) { + super(contents, "SET ", null, null, Arrays.asList(",")); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/SqlNode.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/SqlNode.java new file mode 100644 index 000000000..14cc81468 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/SqlNode.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.orange.node; + +import com.webank.wedatasphere.dss.orange.context.Context; + +import java.util.Set; + + +public interface SqlNode { + + void apply(Context context); + + void applyParameter(Set set); + +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/TextSqlNode.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/TextSqlNode.java new file mode 100644 index 000000000..17a31975d --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/TextSqlNode.java @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.dss.orange.node; + +import com.webank.wedatasphere.dss.orange.context.Context; +import com.webank.wedatasphere.dss.orange.token.TokenHandler; +import com.webank.wedatasphere.dss.orange.token.TokenParser; + +import java.util.Set; + + +public class TextSqlNode implements SqlNode { + + String text; + + public TextSqlNode(String text) { + this.text = text; + } + + @Override + public void apply(Context context) { + //解析常量值 ${xxx} + TokenParser tokenParser = new TokenParser("${", "}", new TokenHandler() { + @Override + public String handleToken(String paramName) { + Object value = context.getOgnlValue(paramName); + return value == null ? "" : value.toString(); + } + }); + String s = tokenParser.parse(text); + + + context.appendSql(s); + + } + + @Override + public void applyParameter(Set set) { + TokenParser tokenParser = new TokenParser("${", "}", new TokenHandler() { + @Override + public String handleToken(String paramName) { + set.add(paramName); + return paramName; + } + }); + String s = tokenParser.parse(text); + + TokenParser tokenParser2 = new TokenParser("#{", "}", new TokenHandler() { + @Override + public String handleToken(String paramName) { + set.add(paramName); + return paramName; + } + }); + tokenParser2.parse(s); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/TrimSqlNode.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/TrimSqlNode.java new file mode 100644 index 000000000..4b1a951dc --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/TrimSqlNode.java @@ -0,0 +1,65 @@ +package com.webank.wedatasphere.dss.orange.node; + +import com.webank.wedatasphere.dss.orange.context.Context; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Set; + + +public class TrimSqlNode implements SqlNode { + + SqlNode contents; + String prefix; + String suffix; + List prefixesToOverride; + List suffixesToOverride; + + public TrimSqlNode(SqlNode contents, String prefix, String suffix, List prefixesToOverride, List suffixesToOverride) { + this.contents = contents; + this.prefix = prefix; + this.suffix = suffix; + this.prefixesToOverride = prefixesToOverride; + this.suffixesToOverride = suffixesToOverride; + } + + @Override + public void apply(Context context) { + context.appendSql(" ");//标签类SqlNode先拼接空格,和前面的内容隔开 + Context proxy = new Context(context.getData()); +// FilterContext filterContext = new FilterContext(context); + contents.apply(proxy); + String sql = proxy.getSql().trim(); + + if (sql.length() > 0) { + if (prefixesToOverride != null) + for (String key : prefixesToOverride) { + if (sql.startsWith(key)) { + sql = sql.substring(key.length()); + } + } + if (suffixesToOverride != null) + for (String key : suffixesToOverride) { + if (sql.endsWith(key)) { + sql = sql.substring(0, sql.length() - key.length()); + } + } + } + + if (StringUtils.isNotBlank(sql) && StringUtils.isNotBlank(prefix)) { + context.appendSql(prefix); + } + + context.appendSql(sql); + + if (StringUtils.isNotBlank(sql) && StringUtils.isNotBlank(suffix)) { + context.appendSql(suffix); + } + + } + + @Override + public void applyParameter(Set set) { + contents.applyParameter(set); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/WhereSqlNode.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/WhereSqlNode.java new file mode 100644 index 000000000..f818b44b4 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/node/WhereSqlNode.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.orange.node; + +import java.util.Arrays; +import java.util.List; + +public class WhereSqlNode extends TrimSqlNode { + + static List prefixesToOverride = Arrays.asList("AND ", "AND\r", "AND\t", "AND\n", "OR ", "OR\r", "OR\t", "OR\n" + , "and ", "and\r", "and\t", "and\n", "or ", "or\r", "or\t", "or\n"); + + public WhereSqlNode(SqlNode contents) { + + super(contents, "WHERE ", null, prefixesToOverride, null); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/ForeachHandler.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/ForeachHandler.java new file mode 100644 index 000000000..83b40c851 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/ForeachHandler.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.dss.orange.tag; + +import com.webank.wedatasphere.dss.orange.node.ForeachSqlNode; +import com.webank.wedatasphere.dss.orange.node.MixedSqlNode; +import com.webank.wedatasphere.dss.orange.node.SqlNode; +import org.apache.commons.lang3.StringUtils; +import org.dom4j.Element; + +import java.util.List; + + +public class ForeachHandler implements TagHandler { + @Override + public void handle(Element element, List targetContents) { + List contents = XmlParser.parseElement(element); + + String open = element.attributeValue("open"); + String close = element.attributeValue("close"); + String collection = element.attributeValue("collection"); + String separator = element.attributeValue("separator"); + String item = element.attributeValue("item"); + String index = element.attributeValue("index"); + + if (StringUtils.isBlank(collection)) { + throw new RuntimeException(" attribute missing : collection"); + } + if (StringUtils.isBlank(item)) { + item = "item"; + } + if (StringUtils.isBlank(index)) { + index = "index"; + } + + ForeachSqlNode foreachSqlNode = new ForeachSqlNode(collection, open, close, separator, item, index, new MixedSqlNode(contents)); + targetContents.add(foreachSqlNode); + + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/IfHandler.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/IfHandler.java new file mode 100644 index 000000000..5b3efda1b --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/IfHandler.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.dss.orange.tag; + +import com.webank.wedatasphere.dss.orange.node.IfSqlNode; +import com.webank.wedatasphere.dss.orange.node.MixedSqlNode; +import com.webank.wedatasphere.dss.orange.node.SqlNode; +import org.dom4j.Element; + +import java.util.List; + + +public class IfHandler implements TagHandler { + + @Override + public void handle(Element element, List targetContents) { + String test = element.attributeValue("test"); + if (test == null) { + throw new RuntimeException(" tag missing test attribute"); + } + + List contents = XmlParser.parseElement(element); + + IfSqlNode ifSqlNode = new IfSqlNode(test, new MixedSqlNode(contents)); + targetContents.add(ifSqlNode); + + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/SetHandler.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/SetHandler.java new file mode 100644 index 000000000..f7ae9ec77 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/SetHandler.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.orange.tag; + +import com.webank.wedatasphere.dss.orange.node.MixedSqlNode; +import com.webank.wedatasphere.dss.orange.node.SetSqlNode; +import com.webank.wedatasphere.dss.orange.node.SqlNode; +import org.dom4j.Element; + +import java.util.List; + + +public class SetHandler implements TagHandler { + + @Override + public void handle(Element element, List targetContents) { + List contents = XmlParser.parseElement(element); + + SetSqlNode node = new SetSqlNode(new MixedSqlNode(contents)); + targetContents.add(node); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/TagHandler.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/TagHandler.java new file mode 100644 index 000000000..97afdc99d --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/TagHandler.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.orange.tag; + +import com.webank.wedatasphere.dss.orange.node.SqlNode; +import org.dom4j.Element; + +import java.util.List; + +public interface TagHandler { + + void handle(Element element, List contents); +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/TrimHandler.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/TrimHandler.java new file mode 100644 index 000000000..789f27db3 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/TrimHandler.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.orange.tag; + +import com.webank.wedatasphere.dss.orange.node.MixedSqlNode; +import com.webank.wedatasphere.dss.orange.node.SqlNode; +import com.webank.wedatasphere.dss.orange.node.TrimSqlNode; +import org.dom4j.Element; + +import java.util.Arrays; +import java.util.List; + + +public class TrimHandler implements TagHandler { + + @Override + public void handle(Element element, List targetContents) { + String prefix = element.attributeValue("prefix"); + String suffix = element.attributeValue("suffix"); + String prefixesToOverride = element.attributeValue("prefixOverrides"); + List prefixesOverride = prefixesToOverride == null ? null : Arrays.asList(prefixesToOverride.split("\\|")); + String suffixesToOverride = element.attributeValue("suffixesOverride"); + List suffixesOverride = suffixesToOverride == null ? null : Arrays.asList(suffixesToOverride.split("\\|")); + + List contents = XmlParser.parseElement(element); + TrimSqlNode trimSqlNode = new TrimSqlNode(new MixedSqlNode(contents), prefix, suffix, prefixesOverride, suffixesOverride); + targetContents.add(trimSqlNode); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/WhereHandler.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/WhereHandler.java new file mode 100644 index 000000000..7cf0aacbf --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/WhereHandler.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.orange.tag; + +import com.webank.wedatasphere.dss.orange.node.MixedSqlNode; +import com.webank.wedatasphere.dss.orange.node.SqlNode; +import com.webank.wedatasphere.dss.orange.node.WhereSqlNode; +import org.dom4j.Element; + +import java.util.List; + + +public class WhereHandler implements TagHandler { + + @Override + public void handle(Element element, List targetContents) { + List contents = XmlParser.parseElement(element); + + WhereSqlNode node = new WhereSqlNode(new MixedSqlNode(contents)); + targetContents.add(node); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/XmlParser.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/XmlParser.java new file mode 100644 index 000000000..b973145b1 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/tag/XmlParser.java @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.dss.orange.tag; + +import com.webank.wedatasphere.dss.orange.node.MixedSqlNode; +import com.webank.wedatasphere.dss.orange.node.SqlNode; +import com.webank.wedatasphere.dss.orange.node.TextSqlNode; +import org.dom4j.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class XmlParser { + + static Map nodeHandlers = new HashMap() { + { + put("foreach", new ForeachHandler()); + put("if", new IfHandler()); + put("trim", new TrimHandler()); + put("where", new WhereHandler()); + put("set", new SetHandler()); + } + }; + + //将xml内容解析成sqlNode类型 + + public static SqlNode parseXml2SqlNode(String text) { + + Document document = null; + try { + document = DocumentHelper.parseText(text); + } catch (DocumentException e) { + throw new RuntimeException(e.getMessage()); + } + Element rootElement = document.getRootElement(); + List contents = parseElement(rootElement); + SqlNode sqlNode = new MixedSqlNode(contents); + return sqlNode; + } + + //解析单个标签的子内容,转化成sqlNode list + + public static List parseElement(Element element) { + List nodes = new ArrayList<>(); + + List children = element.content(); + for (Object node : children) { + if (node instanceof Text) { + TextSqlNode textSqlNode = new TextSqlNode(((Text) node).getText()); + nodes.add(textSqlNode); + + } else if (node instanceof Element) { + String nodeName = ((Element) node).getName(); + TagHandler handler = nodeHandlers.get(nodeName.toLowerCase()); + if (handler == null) { + throw new RuntimeException("tag not supported"); + } + //内部递归调用此方法 + handler.handle((Element) node, nodes); + + } + + } + + return nodes; + + } + + public static void main(String[] args) { + parseXml2SqlNode("111222333444fff555"); + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/token/TokenHandler.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/token/TokenHandler.java new file mode 100644 index 000000000..185d989c0 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/token/TokenHandler.java @@ -0,0 +1,6 @@ +package com.webank.wedatasphere.dss.orange.token; + +public interface TokenHandler { + + public String handleToken(String content); +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/token/TokenParser.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/token/TokenParser.java new file mode 100644 index 000000000..b75cc4ed4 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/token/TokenParser.java @@ -0,0 +1,94 @@ +package com.webank.wedatasphere.dss.orange.token; + +import org.dom4j.DocumentException; + + +public class TokenParser { + + private String openToken; + private String closeToken; + TokenHandler tokenHandler; + + public TokenParser(String openToken, String closeToken, TokenHandler tokenHandler) { + this.openToken = openToken; + this.closeToken = closeToken; + this.tokenHandler = tokenHandler; + } + + public static void main(String[] args) throws DocumentException { +// String parse = parse(" and name = #{minId\\}} and id < #{yy \n} and name = #{ eee }"); +// System.out.println(parse); + +// parseVariableNames("select * from Blog where 1=1 and id > #{minId} and id < #{maxId} \t and id > #{minId} and udr = #{ ffr}"); + } + + /** + * 将sql文本片段中的参数替换成? 并且将?对应的参数值按顺序保存起来 + */ + public String parse(String text) { + if (text == null || text.isEmpty()) { + return ""; + } + int start = text.indexOf(openToken); + if (start == -1) { + return text; + } + char[] src = text.toCharArray(); + int offset = 0; + final StringBuilder builder = new StringBuilder(); + StringBuilder expression = null; + do { + //搜索到假的#{ , \#{ 转化成 #{ + if (start > 0 && src[start - 1] == '\\') { + builder.append(src, offset, start - offset - 1).append(openToken); + offset = start + openToken.length(); + } + //搜索到真实的 #{ + else { + + if (expression == null) { + expression = new StringBuilder(); + } else { + expression.setLength(0); + } + builder.append(src, offset, start - offset); + offset = start + openToken.length(); + + //开始搜索 } + int end = text.indexOf(closeToken, offset); + while (end > -1) { + //搜索到假的 } , \} 转化成 } + if (end > offset && src[end - 1] == '\\') { + expression.append(src, offset, end - offset - 1).append(closeToken); + offset = end + closeToken.length(); + //继续向右搜索 } + end = text.indexOf(closeToken, offset); + } + //搜索到真实的 } + else { + expression.append(src, offset, end - offset); + break; + } + } + //没有搜索到真实的右括号 } + if (end == -1) { + + builder.append(src, start, src.length - start); + offset = src.length; + } + //搜索到真实的右括号} + else { + builder.append(tokenHandler.handleToken(expression.toString().trim())); + offset = end + closeToken.length(); + } + } + start = text.indexOf(openToken, offset); + } while (start > -1); + if (offset < src.length) { + builder.append(src, offset, src.length - offset); + } + return builder.toString(); + } + + +} \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/util/OgnlUtil.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/util/OgnlUtil.java new file mode 100644 index 000000000..b379a0b39 --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/util/OgnlUtil.java @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.dss.orange.util; + +import ognl.Ognl; +import ognl.OgnlException; + +import java.lang.reflect.Array; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class OgnlUtil { + + public static Object getValue(String expression, Map root) { + try { + Map context = Ognl.createDefaultContext(root); + Object value = Ognl.getValue(Ognl.parseExpression(expression), context, root); + return value; + } catch (OgnlException e) { + throw new RuntimeException(e.getMessage()); + } + } + + public static Boolean getBooleanValue(String expression, Map root) { + Object value = getValue(expression, root); + if (value instanceof Boolean) { + return (Boolean) value; + } else if (value instanceof Number) + return !new BigDecimal(String.valueOf(value)).equals(BigDecimal.ZERO); + else + throw new RuntimeException("expression value is not boolean or number type: " + expression); + } + + public static Iterable getIterable(String expression, Map root) { + Object value = getValue(expression, root); + if (value == null) + throw new RuntimeException("The expression '" + expression + "' evaluated to a null value."); + if (value instanceof Iterable) + return (Iterable) value; + if (value.getClass().isArray()) { + // the array may be primitive, so Arrays.asList() may throw + // a ClassCastException (issue 209). Do the work manually + // Curse primitives! :) (JGB) + int size = Array.getLength(value); + List answer = new ArrayList(); + for (int i = 0; i < size; i++) { + Object o = Array.get(value, i); + answer.add(o); + } + return answer; + } + if (value instanceof Map) { + return ((Map) value).entrySet(); + } + throw new RuntimeException("Error evaluating expression '" + expression + "'. Return value (" + value + ") was not iterable."); + } + + public static void main(String[] args) { + Map root = new HashMap<>(); + List list = new ArrayList<>(); + list.add(12); + list.add(22); + list.add(32); + list.add(42); + root.put("ids", list); + + Object o = getValue("ids[3]", root); + System.out.println(o); + + } +} diff --git a/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/util/RegexUtil.java b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/util/RegexUtil.java new file mode 100644 index 000000000..1cbf4e95d --- /dev/null +++ b/dss-apps/dss-data-api/dss-api-sql-template/src/main/java/com/webank/wedatasphere/dss/orange/util/RegexUtil.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.orange.util; + +import java.util.regex.Pattern; + + +public class RegexUtil { + + public static String replace(String content, String item, String newItem) { + return content.replaceFirst("^\\s*" + item + "(?![^.,:\\s])", newItem); + } + + public static void main(String[] args) { + boolean matches = "item".matches( "item" + "[.,:\\s\\[]"); + + boolean item = Pattern.compile("item[.,:\\s\\[]").matcher("item").matches(); + +// String aa = "item[0].name".replaceFirst("^\\s*" + "item" + "(?![^.,:\\s])", "aa"); +// System.out.println(aa); + System.out.println(item); + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/pom.xml b/dss-apps/dss-data-api/dss-data-api-server/pom.xml new file mode 100644 index 000000000..e1d582950 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/pom.xml @@ -0,0 +1,214 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + dss-data-api-server + + + + + + + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + org.springframework.cloud + spring-cloud-netflix + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + + + org.apache.commons + commons-math3 + provided + + + xstream + com.thoughtworks.xstream + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + ${spring.cloud.version} + provided + + + logback-classic + ch.qos.logback + + + log4j-to-slf4j + org.apache.logging.log4j + + + gson + com.google.code.gson + + + jsr311-api + javax.ws.rs + + + xstream + com.thoughtworks.xstream + + + commons-math + org.apache.commons + + + jackson-core + com.fasterxml.jackson.core + + + spring-boot-autoconfigure + org.springframework.boot + + + spring-boot-starter-aop + org.springframework.boot + + + spring-boot-starter + org.springframework.boot + + + spring-boot-starter-cache + org.springframework.boot + + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + + org.projectlombok + lombok + 1.18.16 + true + + + com.alibaba + druid + 1.1.9 + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + provided + + + com.webank.wedatasphere.dss + dss-api-sql-template + ${dss.version} + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + 1.1.1 + provided + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/assembly/distribution.xml b/dss-apps/dss-data-api/dss-data-api-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..4de832a32 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/assembly/distribution.xml @@ -0,0 +1,46 @@ + + + + dss-data-api-server + + dir + + true + dss-data-api-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiAuthMapper.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiAuthMapper.java new file mode 100644 index 000000000..bd19b5751 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiAuthMapper.java @@ -0,0 +1,45 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.api.server.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiAuth; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiAuthInfo; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +import java.util.List; + +@Mapper +public interface ApiAuthMapper extends BaseMapper { + + void addApiAuth(ApiAuth dssDataApiAuth); + + List getApiAuthList(@Param("workspaceId") Long workspaceId, @Param("caller") String caller); + + @Update("UPDATE dss_dataapi_auth SET `is_delete` = 1,`update_time` = NOW() WHERE `id` = #{id}") + void deleteApiAuth(@Param("id") Long id); + + @Select("select UNIX_TIMESTAMP(expire) from dss_dataapi_auth where caller = #{caller} and group_id = #{groupId} and token = #{token}") + Long getToken(@Param("caller") String caller, @Param("groupId") int groupId, @Param("token") String token); + + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiCallMapper.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiCallMapper.java new file mode 100644 index 000000000..582d48cf8 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiCallMapper.java @@ -0,0 +1,97 @@ +package com.webank.wedatasphere.dss.data.api.server.dao; + + +import com.webank.wedatasphere.dss.data.api.server.entity.ApiCall; +import com.webank.wedatasphere.dss.data.api.server.entity.request.CallMonitorResquest; +import com.webank.wedatasphere.dss.data.api.server.entity.request.SingleCallMonitorRequest; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiCallInfoByCnt; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiCallInfoByFailRate; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Options; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; +import java.util.Map; + +public interface ApiCallMapper { + @Insert("INSERT INTO dss_dataapi_call(api_id, params_value, status, time_start, time_end, time_length, caller) " + + "VALUES(#{apiCall.apiId}, #{apiCall.paramsValue}, #{apiCall.status}, #{apiCall.timeStart}, #{apiCall.timeEnd}, #{apiCall.timeLength}, #{apiCall.caller})") + @Options(useGeneratedKeys = true, keyProperty = "apiCall.id", keyColumn = "id") + int addApiCall(@Param("apiCall") ApiCall apiCall); + + @Select("SELECT COUNT(b.id) FROM dss_dataapi_config a\n" + + "LEFT JOIN dss_dataapi_call b ON a.id = b.api_id\n" + + "WHERE a.workspace_id = #{callMonitorResquest.workspaceId}\n" + + " AND (b.time_start BETWEEN #{callMonitorResquest.startTime} AND #{callMonitorResquest.endTime})") + Long getCallTotalCnt(@Param("callMonitorResquest") CallMonitorResquest callMonitorResquest); //总调用次数 + + @Select("SELECT IFNULL(SUM(b.time_length),0) FROM dss_dataapi_config a\n" + + "LEFT JOIN dss_dataapi_call b ON a.id = b.api_id\n" + + "WHERE a.workspace_id = #{callMonitorResquest.workspaceId}\n" + + " AND (b.time_start BETWEEN #{callMonitorResquest.startTime} AND #{callMonitorResquest.endTime})") + Long getCallTotalTime(@Param("callMonitorResquest") CallMonitorResquest callMonitorResquest); //总调用时长 + + + /** + * 调用量排行TOP10 + */ + @Select("SELECT a.id, a.api_name, COUNT(b.id) total_cnt, IFNULL(SUM(b.time_length),0) total_time, IFNULL(ROUND(AVG(b.time_length),0),0) avg_time\n" + + "FROM dss_dataapi_config a\n" + + "LEFT JOIN dss_dataapi_call b ON a.id = b.api_id\n" + + "WHERE a.workspace_id = #{callMonitorResquest.workspaceId}\n" + + " AND (b.time_start BETWEEN #{callMonitorResquest.startTime} AND #{callMonitorResquest.endTime})\n" + + "GROUP BY a.id\n" + + "ORDER BY COUNT(b.id) DESC\n" + + "LIMIT 10") + List getCallListByCnt(@Param("callMonitorResquest") CallMonitorResquest callMonitorResquest); + + /** + * 出错排行TOP10 + */ + @Select("SELECT a.id, a.api_name, COUNT(b.id) total_cnt, SUM(CASE WHEN b.status =2 THEN 1 ELSE 0 END) fail_cnt, ROUND(100 * SUM(CASE WHEN b.status =2 THEN 1 ELSE 0 END) / COUNT(b.id),2) fail_rate\n" + + "FROM dss_dataapi_config a\n" + + "LEFT JOIN dss_dataapi_call b ON a.id = b.api_id\n" + + "WHERE a.workspace_id = #{callMonitorResquest.workspaceId}\n" + + " AND (b.time_start BETWEEN #{callMonitorResquest.startTime} AND #{callMonitorResquest.endTime})\n" + + "GROUP BY a.id\n" + + "ORDER BY fail_rate DESC\n" + + "LIMIT 10") + List getCallListByFailRate(@Param("callMonitorResquest") CallMonitorResquest callMonitorResquest); + + + /** + * 过去24小时,每小时的请求数目 + * 特别注意:dss_project -- 记录数足够的任意一张表,主要是为了给前端拼凑缺失时间段数据 + */ + @Select("SELECT DATE_FORMAT(time_start,'%Y-%m-%d %H:00') AS k, COUNT(*) AS v\n" + + "FROM dss_dataapi_config a JOIN dss_dataapi_call b ON a.id =b.api_id \n" + + "WHERE a.workspace_id =#{workspaceId} AND time_start >= (NOW() - INTERVAL 24 HOUR)\n" + + "GROUP BY DATE_FORMAT(time_start,'%Y-%m-%d %H:00')\n" + + "ORDER BY k") + List> getCallCntForPast24H(Long workspaceId); + + + /** + * 时间范围内指定API的每小时的平均响应时间 + * 特别注意:dss_project -- 记录数足够的任意一张表,主要是为了给前端拼凑缺失时间段数据 + */ + @Select("SELECT DATE_FORMAT(time_start,'%Y-%m-%d %H:00') AS k, IFNULL(ROUND(AVG(time_length),0),0) AS v\n" + + "FROM dss_dataapi_call\n" + + "WHERE api_id =#{singleCallMonitorRequest.apiId} AND (time_start BETWEEN #{singleCallMonitorRequest.startTime} AND #{singleCallMonitorRequest.endTime})\n" + + "GROUP BY DATE_FORMAT(time_start,'%Y-%m-%d %H:00')\n" + + "ORDER BY k") + List> getCallTimeForSinleApi(@Param("singleCallMonitorRequest") SingleCallMonitorRequest singleCallMonitorRequest); + + + /** + * 时间范围内指定API的每小时的请求次数 + * + */ + @Select("SELECT DATE_FORMAT(time_start,'%Y-%m-%d %H:00') AS k, COUNT(id) AS v\n" + + "FROM dss_dataapi_call\n" + + "WHERE api_id =#{singleCallMonitorRequest.apiId} AND (time_start >= #{singleCallMonitorRequest.startTime} AND time_start <= #{singleCallMonitorRequest.endTime})\n" + + "GROUP BY DATE_FORMAT(time_start,'%Y-%m-%d %H:00')\n" + + "ORDER BY k") + List> getCallCntForSinleApi(@Param("singleCallMonitorRequest") SingleCallMonitorRequest singleCallMonitorRequest); +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiConfigMapper.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiConfigMapper.java new file mode 100644 index 000000000..dd4e1057b --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiConfigMapper.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.dss.data.api.server.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiConfig; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiGroup; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiGroupInfo; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiInfo; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiListInfo; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Options; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +import java.util.List; + +public interface ApiConfigMapper extends BaseMapper { + + public Boolean release(@Param("status") Integer status, @Param("apiId") String apiId); + + @Insert("insert into dss_dataapi_group(workspace_id, name, note, create_by) " + + "values(#{apiGroup.workspaceId}, #{apiGroup.name}, #{apiGroup.note}, #{apiGroup.createBy})") + @Options(useGeneratedKeys = true, keyProperty = "apiGroup.id", keyColumn = "id") + int addApiGroup(@Param("apiGroup") ApiGroup apiGroup); + + + + @Select("select id,name from dss_dataapi_group where workspace_id = #{workspaceId}") + @Results({ + @Result(property = "groupId", column = "id"), + @Result(property = "groupName", column = "name"), + + }) + List getGroupByWorkspaceId(@Param("workspaceId") String workspaceId); + + + @Select("select id,api_name as name,req_fields,api_path as path from dss_dataapi_config where group_id = #{groupId}") + List getApiListByGroup(@Param("groupId") int groupId); + + + List getApiInfoList(@Param("workspaceId") Long workspaceId, @Param("apiName") String apiName); + List getOnlineApiInfoList(@Param("workspaceId") Long workspaceId, @Param("apiName") String apiName); + + @Update("UPDATE dss_dataapi_config SET `status` = 0 WHERE `id` = #{apiId}") + void offlineApi(@Param("apiId") Long apiId); + + @Update("UPDATE dss_dataapi_config SET `status` = 1 WHERE `id` = #{apiId}") + void onlineApi(@Param("apiId") Long apiId); + + @Select("SELECT COUNT(1) FROM dss_dataapi_config WHERE `is_delete` = 0 AND `status` = 1 AND `workspace_id` = #{workspaceId}") + Long getOnlineApiCnt(Long workspaceId); + + @Select("SELECT COUNT(1) FROM dss_dataapi_config WHERE `is_delete` = 0 AND `status` = 0 AND `workspace_id` = #{workspaceId}") + Long getOfflineApiCnt(Long workspaceId); + + ApiInfo getApiInfo(Long apiId); + + @Update("UPDATE dss_dataapi_config SET `is_test` = #{isTest} WHERE `id` = #{apiId}") + void updateApiTestStatus(@Param("apiId") int apiId,@Param("isTest") int isTest); +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiGroupMapper.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiGroupMapper.java new file mode 100644 index 000000000..e1e3cf5a9 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/ApiGroupMapper.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.data.api.server.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiGroup; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +@Mapper +public interface ApiGroupMapper extends BaseMapper { + + @Select("SELECT * FROM dss_dataapi_group WHERE workspace_id = #{workspaceId}") + List getApiGroupList(Long workspaceId); +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/DataSourceMapper.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/DataSourceMapper.java new file mode 100644 index 000000000..652d14706 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/DataSourceMapper.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.data.api.server.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.DataSource; + +import java.util.List; + +public interface DataSourceMapper extends BaseMapper { + + List selectByTypeAndWorkspaceId(DataSource dataSource); + + DataSource selectById(Integer datasourceId); + + void addDatasource(DataSource dataSource); + + List listAllDatasources(DataSource dataSource); + + void deleteById(Integer id); + + void editDatasource(DataSource dataSource); + + int dataSourceUsingCount(Integer id); +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiAuthMapper.xml b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiAuthMapper.xml new file mode 100644 index 000000000..89769784c --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiAuthMapper.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiCallMapper.xml b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiCallMapper.xml new file mode 100644 index 000000000..44afb194f --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiCallMapper.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiConfigMapper.xml b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiConfigMapper.xml new file mode 100644 index 000000000..ecc9dc923 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/ApiConfigMapper.xml @@ -0,0 +1,61 @@ + + + + + + + + UPDATE `dss_dataapi_config` + SET `status` = #{status} + WHERE + id = #{apiId} + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/DataSourceMapper.xml b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/DataSourceMapper.xml new file mode 100644 index 000000000..82e4d190f --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/dao/impl/DataSourceMapper.xml @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + insert into dss_dataapi_datasource( + workspace_id, + name, + note, + type, + url, + username, + pwd, + create_by, + create_time + )values( + #{workspaceId}, + #{name}, + #{note}, + #{type}, + #{url}, + #{username}, + #{pwd}, + #{createBy}, + sysdate() + ) + + + + update dss_dataapi_datasource set is_delete = '1' where id = #{id} + + + + update dss_dataapi_datasource + + note = #{note}, + url = #{url}, + username = #{username}, + pwd = #{pwd}, + update_by = #{updateBy}, + update_time = sysdate() + + where id = #{datasourceId} + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiAuth.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiAuth.java new file mode 100644 index 000000000..65add9772 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiAuth.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.dss.data.api.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.webank.wedatasphere.dss.data.api.server.util.DateJsonDeserializer; +import com.webank.wedatasphere.dss.data.api.server.util.DateJsonSerializer; +import lombok.Data; +import org.codehaus.jackson.map.annotate.JsonDeserialize; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.springframework.format.annotation.DateTimeFormat; + +import java.io.Serializable; +import java.util.Date; + +@TableName(value = "dss_dataapi_auth") +@Data +public class ApiAuth implements Serializable { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + private Long workspaceId; + private String caller; + private String token; +// @JsonSerialize(using= DateJsonSerializer.class) +// @JsonDeserialize(using= DateJsonDeserializer.class) + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date expire; + private Long groupId; + +// @JsonSerialize(using= DateJsonSerializer.class) +// @JsonDeserialize(using= DateJsonDeserializer.class) + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + private String createBy; +// @JsonSerialize(using= DateJsonSerializer.class) +// @JsonDeserialize(using= DateJsonDeserializer.class) + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date updateTime; + private String updateBy; + private Integer isDelete; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiCall.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiCall.java new file mode 100644 index 000000000..6e4270c94 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiCall.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.data.api.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.dss.data.api.server.util.DateJsonDeserializer; +import com.webank.wedatasphere.dss.data.api.server.util.DateJsonSerializer; +import lombok.Data; +import org.codehaus.jackson.map.annotate.JsonDeserialize; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.Date; + +@Data +@TableName(value = "dss_dataapi_call") +public class ApiCall { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + private Long apiId; + private String paramsValue; + + private Integer status; + + @JsonSerialize(using= DateJsonSerializer.class) + @JsonDeserialize(using= DateJsonDeserializer.class) + private Date timeStart; + + @JsonSerialize(using= DateJsonSerializer.class) + @JsonDeserialize(using= DateJsonDeserializer.class) + private Date timeEnd; + + private Long timeLength; + private String caller; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiConfig.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiConfig.java new file mode 100644 index 000000000..8e4ea5a16 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiConfig.java @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.dss.data.api.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; + +@Data +@TableName(value = "dss_dataapi_config") +@XmlRootElement +public class ApiConfig { + @TableId(value = "id", type = IdType.AUTO) + Integer id; + @NotBlank(message = "api_tyep不允许为空") + @TableField(value = "api_type") + private String apiType; + @TableField(value = "api_path") + @NotBlank(message = "请求路径不能为空") + private String apiPath; + @NotBlank(message = "api_name 不能为空") + @TableField(value = "api_name") + private String apiName; + private String protocol; + private String previlege; + @TableField("`method`") + private String method; + @TableField("`describe`") + private String describe; + @NotBlank(message = "datasource_id不能为空") + @TableField(value = "datasource_id") + private Integer datasourceId; + private String tblName; + private String memory; + @TableField(value = "req_timeout") + private int reqTimeout; + private String label; + + @TableField(value = "req_fields") + private String reqFields; + + @TableField(value = "order_fields") + private String orderFields; + + private String resFields; + @TableField("`sql`") + + private String sql; + private Integer workspaceId; + private int groupId; + + private String datasourceName; + private String datasourceType; + @TableField(exist = false) + private String resType; + private Integer pageSize; + + private int status; + private int isTest; + private String createBy; + private String updateBy; +} + + + + + + diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiGroup.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiGroup.java new file mode 100644 index 000000000..d25eeea2b --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/ApiGroup.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.data.api.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import javax.validation.constraints.NotBlank; + +@Data +public class ApiGroup { + private Integer id; + @NotBlank(message = "name不能为空") + private String name; + private String note; + private int workspaceId; + private String createBy; + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/DataSource.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/DataSource.java new file mode 100644 index 000000000..ea964aa6b --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/DataSource.java @@ -0,0 +1,151 @@ +package com.webank.wedatasphere.dss.data.api.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.util.Date; + +@Data +@TableName(value = "datasource") +public class DataSource { + + @TableId(value = "id", type = IdType.AUTO) + Integer datasourceId; + + @TableField + Integer workspaceId; + + @TableField + String name; + + @TableField + String note; + + @TableField + String url; + + @TableField + String username; + + @TableField + String pwd; + + @TableField + String type; + + + @TableField + String createBy; + + @TableField + String updateBy; + + + @TableField + Integer isDelete; + + @TableField(exist = false) + String className; + + @TableField + Date createTime; + + public Integer getDatasourceId() { + return datasourceId; + } + + public void setDatasourceId(Integer datasourceId) { + this.datasourceId = datasourceId; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPwd() { + return pwd; + } + + public void setPwd(String pwd) { + this.pwd = pwd; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + public Integer getIsDelete() { + return isDelete; + } + + public void setIsDelete(Integer isDelete) { + this.isDelete = isDelete; + } + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/VariableString.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/VariableString.java new file mode 100644 index 000000000..37b819edd --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/VariableString.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.data.api.server.entity; + + +public class VariableString { + private String path; + + public VariableString(String unparsedPath) { + this.path = unparsedPath; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/CallMonitorResquest.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/CallMonitorResquest.java new file mode 100644 index 000000000..3c7ebe330 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/CallMonitorResquest.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.request; + +import lombok.Data; + +import javax.ws.rs.QueryParam; + +@Data +public class CallMonitorResquest { + @QueryParam("workspaceId") + private Long workspaceId; + + @QueryParam("startTime") + private String startTime; + + + @QueryParam("endTime") + private String endTime; + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/OrderField.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/OrderField.java new file mode 100644 index 000000000..bc7c6a0ea --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/OrderField.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.request; + +import lombok.Data; + +@Data +public class OrderField { + private String name; + private String type; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/ReqField.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/ReqField.java new file mode 100644 index 000000000..76e84f75d --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/ReqField.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.request; + +import lombok.Data; + +@Data +public class ReqField { + private String name; + private String type; + private String compare; + private String comment; + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/SingleCallMonitorRequest.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/SingleCallMonitorRequest.java new file mode 100644 index 000000000..fbe8de82a --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/request/SingleCallMonitorRequest.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.request; + +import lombok.Data; + +import javax.ws.rs.QueryParam; + +@Data +public class SingleCallMonitorRequest { + @QueryParam("apiId") + private Long apiId; + + @QueryParam("startTime") + private String startTime; + + + @QueryParam("endTime") + private String endTime; + + private Long hourCnt; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiAuthInfo.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiAuthInfo.java new file mode 100644 index 000000000..e86352074 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiAuthInfo.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import com.webank.wedatasphere.dss.data.api.server.entity.ApiAuth; +import lombok.Data; + +@Data +public class ApiAuthInfo extends ApiAuth { + private String groupName; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiCallInfoByCnt.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiCallInfoByCnt.java new file mode 100644 index 000000000..db986d766 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiCallInfoByCnt.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import lombok.Data; + +@Data +public class ApiCallInfoByCnt { + private Long id; + private String apiName; + + private Long totalCnt; + private Long totalTime; + private Long avgTime; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiCallInfoByFailRate.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiCallInfoByFailRate.java new file mode 100644 index 000000000..96ced733b --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiCallInfoByFailRate.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import lombok.Data; + +@Data +public class ApiCallInfoByFailRate { + private Long id; + private String apiName; + + private Long totalCnt; + private Long failCnt; + private Long failRate; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiExecuteInfo.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiExecuteInfo.java new file mode 100644 index 000000000..24ec0b93a --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiExecuteInfo.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import lombok.Data; + +@Data +public class ApiExecuteInfo { + private String log; + private ApiResDataInfo resList; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiGroupInfo.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiGroupInfo.java new file mode 100644 index 000000000..ec9ce2180 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiGroupInfo.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import lombok.Data; + +import java.util.List; +@Data +public class ApiGroupInfo { + private int groupId; + private String groupName; + private List apis; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiInfo.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiInfo.java new file mode 100644 index 000000000..5690fb06d --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiInfo.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import com.webank.wedatasphere.dss.data.api.server.util.DateJsonDeserializer; +import com.webank.wedatasphere.dss.data.api.server.util.DateJsonSerializer; +import lombok.Data; +import org.codehaus.jackson.map.annotate.JsonDeserialize; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.Date; + +@Data +public class ApiInfo { + private Long id; + private String apiName; + private String apiPath; + private String apiType; + private Integer status; + private String label; + private String createBy; + @JsonSerialize(using= DateJsonSerializer.class) + @JsonDeserialize(using= DateJsonDeserializer.class) + private Date createTime; + private String updateBy; + @JsonSerialize(using= DateJsonSerializer.class) + @JsonDeserialize(using= DateJsonDeserializer.class) + private Date updateTime; + private String groupName; + private String datasourceName; + private int isTest; + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiListInfo.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiListInfo.java new file mode 100644 index 000000000..998f6e46b --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiListInfo.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import lombok.Data; + +@Data +public class ApiListInfo { + private int id; + private String name; + private String path; + private String reqFields; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiResDataInfo.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiResDataInfo.java new file mode 100644 index 000000000..ab24774f0 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/ApiResDataInfo.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import lombok.Data; + +import java.util.HashMap; +import java.util.List; + +@Data +public class ApiResDataInfo { + private List> data; + private Integer total; + private Integer pageSize; + private Integer currentPageNum; + private Integer totalPage; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/HourMonitorInfo.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/HourMonitorInfo.java new file mode 100644 index 000000000..5f0f1ff4e --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/entity/response/HourMonitorInfo.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.data.api.server.entity.response; + +import lombok.Data; + +@Data +public class HourMonitorInfo { + private String key; + private Object value; + + public HourMonitorInfo(String key,Object value){ + this.key = key; + this.value = value; + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/exception/DataApiException.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/exception/DataApiException.java new file mode 100644 index 000000000..475bc36e4 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/exception/DataApiException.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.dss.data.api.server.exception; + +public class DataApiException extends Exception { + + public DataApiException(final String message) { + super(message); + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiAuthRestful.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiAuthRestful.java new file mode 100644 index 000000000..eb1e5ca16 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiAuthRestful.java @@ -0,0 +1,117 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + + package com.webank.wedatasphere.dss.data.api.server.restful; + + + import com.webank.wedatasphere.dss.data.api.server.entity.ApiAuth; + import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiAuthInfo; + import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiGroupInfo; + import com.webank.wedatasphere.dss.data.api.server.service.ApiAuthService; + import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; + import org.apache.linkis.common.exception.ErrorException; + import org.apache.linkis.server.Message; + import org.apache.linkis.server.security.SecurityFilter; + import lombok.extern.slf4j.Slf4j; + import org.apache.commons.codec.digest.DigestUtils; + import org.springframework.beans.factory.annotation.Autowired; + import org.springframework.web.bind.annotation.*; + + import javax.servlet.http.HttpServletRequest; + import java.util.ArrayList; + import java.util.Date; + import java.util.List; + import java.util.UUID; + + @RestController + @RequestMapping(path = "/dss/data/api/apiauth", produces = {"application/json"}) + @Slf4j + public class DSSDbApiAuthRestful { + @Autowired + private ApiAuthService apiAuthService; + + @RequestMapping(path = "save", method = RequestMethod.POST) + public Message saveApiAuth(HttpServletRequest request, @RequestBody ApiAuth apiAuth) throws ErrorException { + String userName = SecurityFilter.getLoginUsername(request); + if (apiAuth.getId() == null) { + String token = DigestUtils.md5Hex(UUID.randomUUID().toString()); + apiAuth.setToken(token); + + apiAuth.setCreateBy(userName); + apiAuth.setCreateTime(new Date(System.currentTimeMillis())); + apiAuth.setUpdateTime(new Date(System.currentTimeMillis())); + } else { + apiAuth.setUpdateBy(userName); + apiAuth.setUpdateTime(new Date(System.currentTimeMillis())); + } + + boolean flag = apiAuthService.saveApiAuth(apiAuth); + if (flag) { + return Message.ok("保存成功"); + } else { + return Message.error("保存失败"); + } + } + + + @RequestMapping(path = "token", method = RequestMethod.GET) + public Message generateToken() { + String token = DigestUtils.md5Hex(UUID.randomUUID().toString()); + return Message.ok().data("token", token); + } + + + @RequestMapping(path = "list", method = RequestMethod.GET) + public Message getApiAuthList(HttpServletRequest httpServletRequest, @RequestParam(value = "workspaceId", required = false) Long workspaceId, @RequestParam("caller") String caller, + @RequestParam("pageNow") Integer pageNow, @RequestParam("pageSize") Integer pageSize) { + if (pageNow == null) { + pageNow = 1; + } + if (pageSize == null) { + pageSize = 20; + } + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + List totals = new ArrayList<>(); + List apiAuths = apiAuthService.getApiAuthList(Long.valueOf(workspace.getWorkspaceName()), + caller, totals, pageNow, pageSize); + return Message.ok().data("list", apiAuths).data("total", totals.get(0)); + } + + + @RequestMapping(path = "/{id}", method = RequestMethod.POST) + public Message deleteApiAuth(@PathVariable("id") Long id) { + log.info("-------delete apiauth: " + id + ", begin"); + apiAuthService.deleteApiAuth(id); + Message message = Message.ok("删除成功"); + return message; + } + + + @RequestMapping(path = "apigroup", method = RequestMethod.GET) + public Message getApiGroup(HttpServletRequest httpServletRequest, @RequestParam(value = "workspaceId", required = false) Long workspaceId) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + log.info("workspace is {}", workspace.getWorkspaceName()); + + List apiGroupInfoList = apiAuthService.getApiGroupList(Long.valueOf(workspace.getWorkspaceName())); + + Message message = Message.ok().data("list", apiGroupInfoList); + return message; + } + } + diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiConfigRestful.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiConfigRestful.java new file mode 100644 index 000000000..e144cbee2 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiConfigRestful.java @@ -0,0 +1,164 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + + package com.webank.wedatasphere.dss.data.api.server.restful; + + import com.webank.wedatasphere.dss.data.api.server.entity.ApiConfig; + import com.webank.wedatasphere.dss.data.api.server.entity.ApiGroup; + import com.webank.wedatasphere.dss.data.api.server.entity.VariableString; + import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiExecuteInfo; + import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiGroupInfo; + import com.webank.wedatasphere.dss.data.api.server.exception.DataApiException; + import com.webank.wedatasphere.dss.data.api.server.service.ApiConfigService; + import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; + import org.apache.linkis.server.Message; + import org.apache.linkis.server.security.SecurityFilter; + import lombok.extern.slf4j.Slf4j; + import org.codehaus.jettison.json.JSONException; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + import org.springframework.beans.factory.annotation.Autowired; + import org.springframework.web.bind.annotation.*; + + import javax.servlet.http.HttpServletRequest; + import javax.validation.Valid; + import java.util.List; + import java.util.Map; + + + @RestController + @RequestMapping(path = "/dss/data/api", produces = {"application/json"}) + @Slf4j + public class DSSDbApiConfigRestful { + private static final Logger LOGGER = LoggerFactory.getLogger(DSSDbApiConfigRestful.class); + @Autowired + ApiConfigService apiConfigService; + + /** + * 保存api配置信息 + * + * @param request + * @param apiConfig + * @return + */ + @RequestMapping(path = "save", method = RequestMethod.POST) + public Message saveApi(@Valid @RequestBody ApiConfig apiConfig, HttpServletRequest request) throws JSONException, DataApiException { + String username = SecurityFilter.getLoginUsername(request); + apiConfig.setCreateBy(username); + apiConfig.setUpdateBy(username); + apiConfigService.saveApi(apiConfig); + Message message = Message.ok(); + return message; + } + + /** + * 创建API 组 + * + * @param apiGroup + * @return + */ + + @RequestMapping(path = "/group/create", method = RequestMethod.POST) + public Message saveGroup(@Valid @RequestBody ApiGroup apiGroup, HttpServletRequest request) { + String username = SecurityFilter.getLoginUsername(request); + apiGroup.setCreateBy(username); + apiConfigService.addGroup(apiGroup); + Message message = Message.ok().data("groupId", apiGroup.getId()); + return message; + } + + /** + * API list + * + * @param workspaceId + * @return + */ + + @RequestMapping(path = "list", method = RequestMethod.GET) + public Message getApiList(HttpServletRequest httpServletRequest, + @RequestParam(value = "workspaceId", required = false) String workspaceId) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + + List list = apiConfigService.getGroupList(workspace.getWorkspaceName()); + Message message = Message.ok().data("list", list); + return message; + } + + /** + * 查询api详情 + * + * @param apiId + * @return + */ + + @RequestMapping(path = "detail", method = RequestMethod.GET) + public Message getApiDetail(@RequestParam("apiId") int apiId) { + ApiConfig apiConfig = apiConfigService.getById(apiId); + Message message = Message.ok().data("detail", apiConfig); + return message; + } + + /** + * 测试 API + * + * @param request + * @param path + * @param map + * @return + */ + + @RequestMapping(value = "/test/{path:[a-zA-Z0-9_-]+}", method = RequestMethod.POST) + public Message testApi(HttpServletRequest request, @PathVariable("path") VariableString path, + @RequestBody Map map) { + + try { + ApiExecuteInfo resJo = apiConfigService.apiTest(path.getPath(), request, map, true); + Message message = Message.ok().data("response", resJo); + return message; + } catch (Exception exception) { + log.error("ERROR", "Error found: ", exception); + return Message.error(exception.getMessage()); + } + + } + + + /** + * 第三方调用 api + * + * @param request + * @param path + * @param map + * @return + */ + + @RequestMapping(value = "/execute/{path:[a-zA-Z0-9_-]+}", method = RequestMethod.POST) + public Message executeApi(HttpServletRequest request, @PathVariable("path") VariableString path, @RequestBody Map map) { + try { + ApiExecuteInfo resJo = apiConfigService.apiExecute(path.getPath(), request, map); + Message message = Message.ok().data("response", resJo); + return message; + } catch (Exception exception) { + log.error("ERROR", "Error found: ", exception); + return Message.error(exception.getMessage()); + } + + } + + } diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiDataSourceRestful.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiDataSourceRestful.java new file mode 100644 index 000000000..2da04f30a --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiDataSourceRestful.java @@ -0,0 +1,129 @@ +package com.webank.wedatasphere.dss.data.api.server.restful; + +import com.alibaba.druid.pool.DruidPooledConnection; +import com.webank.wedatasphere.dss.data.api.server.entity.DataSource; +import com.webank.wedatasphere.dss.data.api.server.service.ApiDataSourceService; +import com.webank.wedatasphere.dss.data.api.server.util.CryptoUtils; +import com.webank.wedatasphere.dss.data.api.server.util.JdbcUtil; +import com.webank.wedatasphere.dss.data.api.server.util.PoolManager; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; + +@RestController +@RequestMapping(path = "/dss/data/api/datasource", produces = {"application/json"}) +public class DSSDbApiDataSourceRestful { + + @Resource + private ApiDataSourceService dssDbApiDataSourceService; + private static final Logger logger = LoggerFactory.getLogger(DSSDbApiDataSourceRestful.class); + + @RequestMapping(path = "connections", method = RequestMethod.GET) + public Message connect(HttpServletRequest httpServletRequest, @RequestParam(value = "workspaceId", required = false) Integer workspaceId, + @RequestParam("type") String type) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + List allConnections = dssDbApiDataSourceService.getAllConnections(Integer.valueOf(workspace.getWorkspaceName()), type); + return Message.ok().data("availableConns", allConnections); + + } + + + @RequestMapping(path = "tables", method = RequestMethod.GET) + public Message getAllTables(@RequestParam("datasourceId") Integer datasourceId) throws SQLException { + + DataSource dataSource = dssDbApiDataSourceService.selectById(datasourceId); + DruidPooledConnection connection = PoolManager.getPooledConnection(dataSource); + List tables = JdbcUtil.getAllTables(connection, dataSource.getType()); + return Message.ok().data("allTables", tables); + } + + + @RequestMapping(path = "cols", method = RequestMethod.GET) + public Message getAllCols(@RequestParam("datasourceId") Integer datasourceId, @RequestParam("tableName") String tableName) throws SQLException { + DataSource dataSource = dssDbApiDataSourceService.selectById(datasourceId); + DruidPooledConnection connection = PoolManager.getPooledConnection(dataSource); + List columns = JdbcUtil.getRDBMSColumnProperties(connection, dataSource.getType(), tableName); + return Message.ok().data("allCols", columns); + } + + + @RequestMapping(path = "add", method = RequestMethod.POST) + public Message addDatasource(@RequestBody DataSource dataSource, HttpServletRequest req) { + + dataSource.setPwd(CryptoUtils.object2String(dataSource.getPwd())); + dataSource.setCreateBy(SecurityFilter.getLoginUsername(req)); + dssDbApiDataSourceService.addDatasource(dataSource); + return Message.ok("保存成功"); + } + + + @RequestMapping(path = "list", method = RequestMethod.GET) + public Message getAllDs(HttpServletRequest httpServletRequest, + @RequestParam(value = "workspaceId", required = false) Integer workspaceId, + @RequestParam("type") String type, @RequestParam("name") String name) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + DataSource dataSource = new DataSource(); + dataSource.setWorkspaceId(Integer.valueOf(workspace.getWorkspaceName())); + dataSource.setType(type); + dataSource.setName(name); + List allDatasource = dssDbApiDataSourceService.listAllDatasources(dataSource); + + return Message.ok().data("allDs", allDatasource); + } + + @RequestMapping(path = "edit", method = RequestMethod.POST) + public Message editDatasource(@RequestBody DataSource dataSource, HttpServletRequest req) { + if (StringUtils.isNotEmpty(dataSource.getPwd())) { + PoolManager.removeJdbcConnectionPool(dataSource.getDatasourceId()); + dataSource.setPwd(CryptoUtils.object2String(dataSource.getPwd())); + dataSource.setUpdateBy(SecurityFilter.getLoginUsername(req)); + dssDbApiDataSourceService.editDatasource(dataSource); + return Message.ok("修改成功"); + } + return Message.error("密码不能为空"); + } + + + @RequestMapping(path = "/delete/{id}", method = RequestMethod.POST) + public Message deleteDatasource(@PathVariable("id") Integer id) { + if (dssDbApiDataSourceService.isDataSourceUsing(id)) { + return Message.error("该数据源正在被使用,请下线与此数据源相关的api后再删除"); + } else { + dssDbApiDataSourceService.deleteById(id); + return Message.ok("删除成功"); + } + } + + + @RequestMapping(path = "test", method = RequestMethod.POST) + public Message testDatasource(@RequestBody DataSource dataSource) { + Connection connection = null; + try { + connection = JdbcUtil.getConnection(dataSource); + } catch (Exception e) { + return Message.error(e.getMessage()); + } finally { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + logger.error(e.getMessage()); + } + } + } + return Message.ok("测试连接成功"); + } + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiManagerRestful.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiManagerRestful.java new file mode 100644 index 000000000..0c25c3432 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiManagerRestful.java @@ -0,0 +1,84 @@ +package com.webank.wedatasphere.dss.data.api.server.restful; + +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiInfo; +import com.webank.wedatasphere.dss.data.api.server.exception.DataApiException; +import com.webank.wedatasphere.dss.data.api.server.service.ApiManagerService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.linkis.server.Message; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.List; + +@RestController +@RequestMapping(path = "/dss/data/api/apimanager", produces = {"application/json"}) +public class DSSDbApiManagerRestful { + @Autowired + private ApiManagerService apiManagerService; + + + @RequestMapping(path = "list", method = RequestMethod.GET) + public Message getApiList(HttpServletRequest request, + @RequestParam(value = "workspaceId", required = false) Long workspaceId, + @RequestParam("apiName") String apiName, @RequestParam("pageNow") Integer pageNow, + @RequestParam("pageSize") Integer pageSize) { + if (pageNow == null) { + pageNow = 1; + } + if (pageSize == null) { + pageSize = 20; + } + Workspace workspace = SSOHelper.getWorkspace(request); + List totals = new ArrayList<>(); + List apiInfoList = apiManagerService.getApiInfoList(Long.valueOf(workspace.getWorkspaceName()) + , apiName, totals, pageNow, pageSize); + return Message.ok().data("list", apiInfoList).data("total", totals.get(0)); + } + + + @RequestMapping(path = "/offline/{apiId}", method = RequestMethod.POST) + public Message offlineApi(@PathVariable("apiId") Long apiId) { + apiManagerService.offlineApi(apiId); + ApiInfo apiInfo = apiManagerService.getApiInfo(apiId); + + Message message = Message.ok("下线API成功").data("apiInfo", apiInfo); + return message; + } + + + @RequestMapping(path = "/online/{apiId}", method = RequestMethod.POST) + public Message onlineApi(@PathVariable("apiId") Long apiId) throws DataApiException { + + + ApiInfo apiInfo = apiManagerService.getApiInfo(apiId); + if (apiInfo.getIsTest() == 0) { + throw new DataApiException("请测试通过后再上线"); + } + + if (apiInfo.getStatus() == 1) { + throw new DataApiException("该Api已发布,请勿重复发布"); + } + + apiManagerService.onlineApi(apiId); + apiInfo = apiManagerService.getApiInfo(apiId); + Message message = Message.ok("上线API成功").data("apiInfo", apiInfo); + return message; + } + + + @RequestMapping(path = "/callPath/{apiId}", method = RequestMethod.GET) + public Message getApiCallPath(@PathVariable("apiId") Long apiId) { + StringBuilder callPath = new StringBuilder("{protocol}://{host}"); + callPath.append("/api/rest_j/v1/dss/data/api/execute"); + ApiInfo apiInfo = apiManagerService.getApiInfo(apiId); + callPath.append("/" + apiInfo.getApiPath()); + + Message message = Message.ok().data("callPathPrefix", callPath.toString()); + return message; + } + + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiMonitorRestful.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiMonitorRestful.java new file mode 100644 index 000000000..ee1d9cd8d --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/restful/DSSDbApiMonitorRestful.java @@ -0,0 +1,128 @@ +package com.webank.wedatasphere.dss.data.api.server.restful; + +import com.webank.wedatasphere.dss.data.api.server.entity.request.CallMonitorResquest; +import com.webank.wedatasphere.dss.data.api.server.entity.request.SingleCallMonitorRequest; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiInfo; +import com.webank.wedatasphere.dss.data.api.server.service.ApiManagerService; +import com.webank.wedatasphere.dss.data.api.server.service.ApiMonitorService; +import com.webank.wedatasphere.dss.data.api.server.util.TimeUtil; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.BeanParam; +import java.util.ArrayList; +import java.util.List; + +@RestController +@RequestMapping(path = "/dss/data/api/apimonitor", produces = {"application/json"}) +public class DSSDbApiMonitorRestful { + public static final Logger LOGGER = LoggerFactory.getLogger(DSSDbApiMonitorRestful.class); + + @Autowired + private ApiMonitorService apiMonitorService; + @Autowired + private ApiManagerService apiManagerService; + + @RequestMapping(path = "list", method = RequestMethod.GET) + public Message getApiList(HttpServletRequest request, + @RequestParam(value = "workspaceId", required = false) Long workspaceId, @RequestParam(value = "apiName", required = false) String apiName, + @RequestParam("pageNow") Integer pageNow, @RequestParam("pageSize") Integer pageSize) { + if (pageNow == null) { + pageNow = 1; + } + if (pageSize == null) { + pageSize = 20; + } + Workspace workspace = SSOHelper.getWorkspace(request); + LOGGER.info("workspace is: {}", workspace.getWorkspaceName()); + List totals = new ArrayList<>(); + List apiInfoList = apiManagerService.getOnlineApiInfoList(Long.valueOf(workspace.getWorkspaceName()), + apiName, totals, pageNow, pageSize); + return Message.ok().data("list", apiInfoList).data("total", totals.get(0)); + } + + + @RequestMapping(path = "onlineApiCnt", method = RequestMethod.GET) + public Message getOnlineApiCnt(HttpServletRequest request, @RequestParam(value = "workspaceId", required = false) Long workspaceId) { + Workspace workspace = SSOHelper.getWorkspace(request); + LOGGER.info("workspace is: {}", workspace.getWorkspaceName()); + return Message.ok().data("onlineApiCnt", apiMonitorService.getOnlineApiCnt(Long.valueOf(workspace.getWorkspaceName()))); + } + + + @RequestMapping(path = "offlineApiCnt", method = RequestMethod.GET) + public Message getOfflineApiCnt(HttpServletRequest request, @RequestParam(value = "workspaceId", required = false) Long workspaceId) { + Workspace workspace = SSOHelper.getWorkspace(request); + LOGGER.info("workspace is: {}", workspace.getWorkspaceName()); + return Message.ok().data("offlineApiCnt", apiMonitorService.getOfflineApiCnt(Long.valueOf(workspace.getWorkspaceName()))); + } + + + @RequestMapping(path = "callTotalCnt", method = RequestMethod.GET) + public Message getCallTotalCnt(@BeanParam CallMonitorResquest callMonitorResquest) { + return Message.ok().data("callTotalCnt", apiMonitorService.getCallTotalCnt(callMonitorResquest)); + } + + + @RequestMapping(path = "callTotalTime", method = RequestMethod.GET) + public Message getCallTotalTime(@BeanParam CallMonitorResquest callMonitorResquest) { + return Message.ok().data("callTotalTime", apiMonitorService.getCallTotalTime(callMonitorResquest)); + } + + + @RequestMapping(path = "callListByCnt", method = RequestMethod.GET) + public Message getCallListByCnt(@BeanParam CallMonitorResquest callMonitorResquest) { + return Message.ok().data("list", apiMonitorService.getCallListByCnt(callMonitorResquest)); + } + + + @RequestMapping(path = "callListByFailRate", method = RequestMethod.GET) + public Message getCallListByFailRate(@BeanParam CallMonitorResquest callMonitorResquest) { + return Message.ok().data("list", apiMonitorService.getCallListByFailRate(callMonitorResquest)); + } + + /** + * 过去24小时内每小时请求次数(平均QPS) + */ + + @RequestMapping(path = "callCntForPast24H", method = RequestMethod.GET) + public Message getCallCntForPast24H(HttpServletRequest request, @RequestParam(value = "workspaceId", required = false) Long workspaceId) + throws Exception { + Workspace workspace = SSOHelper.getWorkspace(request); + LOGGER.info("workspace is: {}", workspace.getWorkspaceName()); + return Message.ok().data("list", apiMonitorService.getCallCntForPast24H(Long.valueOf(workspace.getWorkspaceName()))); + } + + /** + * 单个API每小时的平均响应时间 + */ + + @RequestMapping(path = "callTimeForSinleApi", method = RequestMethod.GET) + public Message getCallTimeForSinleApi(@BeanParam SingleCallMonitorRequest singleCallMonitorRequest) throws Exception { + long hourCnt = TimeUtil.getHourCnt(singleCallMonitorRequest.getStartTime(), singleCallMonitorRequest.getEndTime()); + singleCallMonitorRequest.setHourCnt(hourCnt); + + return Message.ok().data("list", apiMonitorService.getCallTimeForSinleApi(singleCallMonitorRequest)); + } + + /** + * 单个API每小时的调用次数 + */ + + @RequestMapping(path = "callCntForSinleApi", method = RequestMethod.GET) + public Message getCallCntForSinleApi(@BeanParam SingleCallMonitorRequest singleCallMonitorRequest) throws Exception { + long hourCnt = TimeUtil.getHourCnt(singleCallMonitorRequest.getStartTime(), singleCallMonitorRequest.getEndTime()); + singleCallMonitorRequest.setHourCnt(hourCnt); + + return Message.ok().data("list", apiMonitorService.getCallCntForSinleApi(singleCallMonitorRequest)); + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiAuthService.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiAuthService.java new file mode 100644 index 000000000..c1027bba3 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiAuthService.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.data.api.server.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiAuth; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiAuthInfo; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiGroupInfo; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.List; + +public interface ApiAuthService extends IService { + public List getApiGroupList(Long workspaceId); + public boolean saveApiAuth(ApiAuth apiAuth) throws ErrorException; + + public List getApiAuthList(Long workspaceId, String caller, List totals, Integer pageNow, Integer pageSize); + + public void deleteApiAuth(Long id); + + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiConfigService.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiConfigService.java new file mode 100644 index 000000000..5bcfa1211 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiConfigService.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.data.api.server.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.google.gson.JsonObject; + +import com.webank.wedatasphere.dss.data.api.server.entity.ApiConfig; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiGroup; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiExecuteInfo; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiGroupInfo; +import com.webank.wedatasphere.dss.data.api.server.exception.DataApiException; +import org.codehaus.jettison.json.JSONException; + +import javax.servlet.http.HttpServletRequest; +import java.util.List; +import java.util.Map; + +public interface ApiConfigService extends IService { + + Boolean release(Integer status, String apiId); + + void addGroup(ApiGroup apiGroup); + + List getGroupList(String workspaceId); + + void saveApi(ApiConfig apiConfig) throws JSONException, DataApiException; + + ApiExecuteInfo apiTest(String path, HttpServletRequest httpRequest, Map map,boolean isTest) throws Exception; + + ApiExecuteInfo apiExecute(String path, HttpServletRequest request, Map map) throws Exception; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiDataSourceService.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiDataSourceService.java new file mode 100644 index 000000000..77ab2b260 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiDataSourceService.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.data.api.server.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.data.api.server.entity.DataSource; + +import java.util.List; + +public interface ApiDataSourceService extends IService { + + DataSource selectById(Integer datasourceId); + + List getAllConnections(Integer workspaceId, String type); + + + + List getAvailableConns(List allConnections); + + void addDatasource(DataSource dataSource); + + + List listAllDatasources(DataSource dataSource); + + void deleteById(Integer datasourceId); + + void editDatasource(DataSource dataSource); + + boolean isDataSourceUsing(Integer id); +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiManagerService.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiManagerService.java new file mode 100644 index 000000000..1ab1db8b7 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiManagerService.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.data.api.server.service; + + +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiInfo; + +import java.util.List; + +public interface ApiManagerService { + //public List getApiGroupList(Long workspaceId); + + public ApiInfo getApiInfo(Long apiId); + public List getApiInfoList(Long workspaceId, String apiName, List totals, Integer pageNow, Integer pageSize); + public List getOnlineApiInfoList(Long workspaceId, String apiName, List totals, Integer pageNow, Integer pageSize); + + public void offlineApi(Long apiId); + + public void onlineApi(Long apiId); +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiMonitorService.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiMonitorService.java new file mode 100644 index 000000000..1bdcc4898 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/ApiMonitorService.java @@ -0,0 +1,58 @@ +package com.webank.wedatasphere.dss.data.api.server.service; + + + +import com.webank.wedatasphere.dss.data.api.server.entity.request.CallMonitorResquest; +import com.webank.wedatasphere.dss.data.api.server.entity.request.SingleCallMonitorRequest; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiCallInfoByCnt; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiCallInfoByFailRate; +import com.webank.wedatasphere.dss.data.api.server.entity.response.HourMonitorInfo; + +import java.util.List; + +public interface ApiMonitorService { + /** + * 已发布API数量 + */ + Long getOnlineApiCnt(Long workspaceId); + + /** + * 未发布API数量 + */ + Long getOfflineApiCnt(Long workspaceId); + + /** + * 总调用次数 + */ + Long getCallTotalCnt(CallMonitorResquest callMonitorResquest); + + /** + * 总执行时长 + */ + Long getCallTotalTime(CallMonitorResquest callMonitorResquest); + + /** + * 出错率排行TOP10 + */ + List getCallListByCnt(CallMonitorResquest callMonitorResquest); + + /** + * 调用量排行TOP10 + */ + List getCallListByFailRate(CallMonitorResquest callMonitorResquest); + + /** + * 过去24小时,每小时的请求数目 + */ + List getCallCntForPast24H(Long workspaceId) throws Exception; + + /** + * 时间范围内指定API的每小时的平均响应时间 + */ + List getCallTimeForSinleApi(SingleCallMonitorRequest singleCallMonitorRequest) throws Exception; + + /** + * 时间范围内指定API的每小时的请求次数 + */ + List getCallCntForSinleApi(SingleCallMonitorRequest singleCallMonitorRequest) throws Exception; +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiAuthServiceImpl.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiAuthServiceImpl.java new file mode 100644 index 000000000..307185553 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiAuthServiceImpl.java @@ -0,0 +1,68 @@ +package com.webank.wedatasphere.dss.data.api.server.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; + +import com.webank.wedatasphere.dss.data.api.server.dao.ApiAuthMapper; +import com.webank.wedatasphere.dss.data.api.server.dao.ApiConfigMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiAuth; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiAuthInfo; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiGroupInfo; +import com.webank.wedatasphere.dss.data.api.server.service.ApiAuthService; +import org.apache.linkis.common.exception.ErrorException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@Slf4j +public class ApiAuthServiceImpl extends ServiceImpl implements ApiAuthService { + @Autowired + private ApiAuthMapper apiAuthMapper; + @Autowired + private ApiConfigMapper apiConfigMapper; + + @Override + public boolean saveApiAuth(ApiAuth apiAuth) throws ErrorException { + Long id = apiAuth.getId(); + + if(id != null){ + return this.updateById(apiAuth); + } + else { + return this.save(apiAuth); + } + } + + @Override + public List getApiAuthList(Long workspaceId, String caller, List totals, Integer pageNow, Integer pageSize){ + PageHelper.startPage(pageNow, pageSize, true); + // MYSQL LIKE % _: LIKE '%\_%', LIKE '%\%%' + if(caller !=null) { + if ("_".equalsIgnoreCase(caller.trim())) { + caller = "\\_"; + } + if ("%".equalsIgnoreCase(caller.trim())) { + caller = "\\%"; + } + } + List apiAuthList = apiAuthMapper.getApiAuthList(workspaceId,caller); + PageInfo pageInfo = new PageInfo<>(apiAuthList); + totals.add(pageInfo.getTotal()); + + return apiAuthList; + } + + @Override + public void deleteApiAuth(Long id){ + apiAuthMapper.deleteApiAuth(id); + } + + @Override + public List getApiGroupList(Long workspaceId){ + return apiConfigMapper.getGroupByWorkspaceId(workspaceId.toString()); + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiConfigServiceImpl.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiConfigServiceImpl.java new file mode 100644 index 000000000..18e07d8c5 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiConfigServiceImpl.java @@ -0,0 +1,364 @@ +package com.webank.wedatasphere.dss.data.api.server.service.impl; + +import com.alibaba.druid.pool.DruidPooledConnection; +import com.alibaba.druid.sql.PagerUtils; +import com.baomidou.mybatisplus.annotation.DbType; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.data.api.server.dao.ApiAuthMapper; +import com.webank.wedatasphere.dss.data.api.server.dao.ApiCallMapper; +import com.webank.wedatasphere.dss.data.api.server.dao.ApiConfigMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiCall; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiConfig; +import com.webank.wedatasphere.dss.data.api.server.entity.ApiGroup; +import com.webank.wedatasphere.dss.data.api.server.entity.DataSource; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiExecuteInfo; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiGroupInfo; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiResDataInfo; +import com.webank.wedatasphere.dss.data.api.server.exception.DataApiException; +import com.webank.wedatasphere.dss.data.api.server.service.ApiConfigService; +import com.webank.wedatasphere.dss.data.api.server.service.ApiDataSourceService; +import com.webank.wedatasphere.dss.data.api.server.util.*; +import com.webank.wedatasphere.dss.orange.SqlMeta; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import javax.servlet.http.HttpServletRequest; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class ApiConfigServiceImpl extends ServiceImpl implements ApiConfigService { + @Autowired + ApiConfigMapper dssApiConfigMapper; + @Autowired + ApiAuthMapper apiAuthMapper; + @Autowired + ApiCallMapper apiCallMapper; + @Autowired + ApiConfigMapper apiConfigMapper; + @Autowired + ApiDataSourceService apiDataSourceService; + + /** + * 保存API配置信息 + * @param apiConfig + * @throws JSONException + */ + public void saveApi(ApiConfig apiConfig) throws JSONException, DataApiException { + + List apiList = this.list(new QueryWrapper().eq("api_path", apiConfig.getApiPath())); + Integer id = apiConfig.getId(); + if(apiList.size() > 0 && id == null){ + throw new DataApiException("路径已经存在"); + } + String apiType = apiConfig.getApiType(); + if ("GUIDE".equals(apiType)) { + String table = apiConfig.getTblName(); + String resFields = apiConfig.getResFields(); + String reqFields = apiConfig.getReqFields(); + String orderFields = apiConfig.getOrderFields(); + String whereCause = StringUtils.isBlank(reqFields) ? "" : CommUtil.getWhereCause(reqFields); + String orderCause = StringUtils.isBlank(orderFields) ? "" : CommUtil.getOrderCause(orderFields); + String sql = String.format("%s%s%s%s%s%s", "select ", resFields, " from ", table, whereCause, orderCause); + apiConfig.setSql(sql); + } +// UpdateWrapper apiConfigUpdateWrapper = new UpdateWrapper() +// .eq("id", id); + if (id != null) { + if(apiConfig.getStatus() == 1){ + throw new DataApiException("请先下线,测试通过后, 重新发布"); + } + apiConfig.setIsTest(0); + this.updateById(apiConfig); + } else { + this.save(apiConfig); + } + } + + /** + * API 测试 + * @param path + * @param request + * @param map + * @return + * @throws Exception + */ + + @Override + public ApiExecuteInfo apiTest(String path, HttpServletRequest request,Map map,boolean isTest) throws JSONException, SQLException, DataApiException { + ApiExecuteInfo apiExecuteInfo = new ApiExecuteInfo(); + ApiConfig apiConfig = this.getOne(new QueryWrapper().eq("api_path", path)); + List jdbcParamValues = new ArrayList<>(); + int pageSize = apiConfig.getPageSize(); + String limitSent = ""; + String sqlText = ""; + int pageNum = 0; + + //组装分页limit + if(pageSize > 0 ){ + Object pageNumObject = map.get("pageNum"); + if(pageNumObject == null){ + throw new DataApiException("请设置pageNum参数"); + } + pageNum = Integer.valueOf(pageNumObject.toString()); + if(pageNum < 1){ + throw new DataApiException("pageNum参数错误"); + } + limitSent = " limit "+ ((pageNum-1) * pageSize)+","+pageSize; + } + if (apiConfig != null) { + Map sqlParam = this.getSqlParam(request, apiConfig,map); + String sqlFiled = apiConfig.getSql(); + if(!sqlParam.isEmpty()){ + SqlMeta sqlMeta = SqlEngineUtil.getEngine().parse(sqlFiled, sqlParam); + sqlText = sqlMeta.getSql(); + jdbcParamValues = sqlMeta.getJdbcParamValues(); + }else { + sqlText = sqlFiled; + } + //不分页,最多返回500条数据 + if(pageSize <= 0) { + sqlText = String.format("%s %s",sqlText," limit 500"); + } + + Integer datasourceId = apiConfig.getDatasourceId(); + DataSource dataSource = apiDataSourceService.getById(datasourceId); + + String dataSourceType = dataSource.getType(); + if("MYSQL".equals(dataSourceType)){ + dataSource.setClassName("com.mysql.jdbc.Driver"); + } + if(sqlText.toLowerCase().contains("select")){ + apiExecuteInfo = this.executeSql(1, dataSource, sqlText,limitSent, jdbcParamValues,pageSize,pageNum); + }else { + apiExecuteInfo = this.executeSql(2, dataSource, sqlText,limitSent, jdbcParamValues,pageSize,pageNum); + } + if(isTest){ + apiConfigMapper.updateApiTestStatus(apiConfig.getId(),1); + } + + }else { + apiExecuteInfo.setLog("该服务不存在,请检查服务url是否正确"); + } + return apiExecuteInfo; + } + + + /** + * 第三方调用APi + * @param path + * @param request + * @return + * @throws Exception + */ + @Override + public ApiExecuteInfo apiExecute(String path, HttpServletRequest request,Map map) throws DataApiException, JSONException, SQLException { + ApiCall apiCall = new ApiCall(); + //校验token + String appKey = request.getHeader("appKey"); + String appSecret = request.getHeader("appSecret"); + if(StringUtils.isAnyBlank(appKey,appSecret)){ + throw new DataApiException("请求header需添加appKey,appSecret"); + } + ApiConfig apiConfig = this.getOne(new QueryWrapper().eq("api_path", path).eq("is_delete",0)); + if(apiConfig != null){ + int status = apiConfig.getStatus(); + if(status == 0){ + throw new DataApiException("该服务已下线"); + } + long startTime = System.currentTimeMillis(); + apiCall.setApiId(apiConfig.getId().longValue()); + apiCall.setTimeStart(new Date(startTime)); + apiCall.setCaller(appKey); + int groupId = apiConfig.getGroupId(); + Long expireTime = apiAuthMapper.getToken(appKey,groupId,appSecret); + if(expireTime != null && (expireTime * 1000) > startTime){ + ApiExecuteInfo apiExecuteInfo = apiTest(path,request,map,false); + long endTime = System.currentTimeMillis(); + apiCall.setTimeEnd(new Date(endTime)); + apiCall.setTimeLength(endTime-startTime); + apiCall.setStatus(0); + apiCallMapper.addApiCall(apiCall); + return apiExecuteInfo; + }else { + throw new DataApiException("token已失效"); + } + }else { + throw new DataApiException("该服务不存在,请检查服务url是否正确"); + } + } + + + + @Override + public void addGroup(ApiGroup apiGroup) { + dssApiConfigMapper.addApiGroup(apiGroup); + } + + @Override + public List getGroupList(String workspaceId) { + List apiGroupInfoList = dssApiConfigMapper.getGroupByWorkspaceId(workspaceId); + for (ApiGroupInfo apiGroupInfo : apiGroupInfoList) { + int groupId = apiGroupInfo.getGroupId(); + apiGroupInfo.setApis(dssApiConfigMapper.getApiListByGroup(groupId)); + } + return apiGroupInfoList; + } + + + @Override + public Boolean release(Integer status, String apiId) { + return apiConfigMapper.release(status,apiId); + } + + + private Map getSqlParam(HttpServletRequest request, ApiConfig config,Map paraMap) throws JSONException { + Map map = new HashMap<>(); + String reqFields = config.getReqFields(); + if(StringUtils.isNotBlank(reqFields)){ + JSONArray requestParams = new JSONArray(config.getReqFields()); + for (int i = 0; i < requestParams.length(); i++) { + JSONObject jo = requestParams.getJSONObject(i); + String name = jo.getString("name"); + String type = jo.getString("type"); + if(type.indexOf("(") > 0){ + type = type.substring(0,type.indexOf("(")).toLowerCase(); + } + if (type.startsWith("Array")) { + String[] values = CommUtil.objectToArray(paraMap.get(name)); + if (values != null) { + List list = Arrays.asList(values); + if (values.length > 0) { + switch (type) { + case "Array": + List collect = list.stream().map(value -> Double.valueOf(value)).collect(Collectors.toList()); + map.put(name, collect); + break; + case "Array": + List longs = list.stream().map(value -> Long.valueOf(value)).collect(Collectors.toList()); + map.put(name, longs); + break; +// case "Array": +// case "Array": + default: + map.put(name, list); + break; + } + } else { + map.put(name, list); + } + } else { + map.put(name, null); + } + } else { + + String value = paraMap == null || paraMap.get(name) == null ? null : String.valueOf(paraMap.get(name)); + if (StringUtils.isNotBlank(value)) { + + switch (type) { + case "double": + Double v = Double.valueOf(value); + map.put(name, v); + break; + case "int": + case "bigint": + Long longV = Long.valueOf(value); + map.put(name, longV); + break; +// case "string": +// case "varchar": +// case "datetime": +// case "timestamp": +// case "date": + default: + map.put(name, value); + break; + } + } else { + map.put(name, value); + } + } + } + + } + + return map; + } + + + public ApiExecuteInfo executeSql(int isSelect, DataSource datasource, String sql,String pageSent, List jdbcParamValues,int pageSize,int pageNum) throws DataApiException, SQLException { + DruidPooledConnection connection = null; + StringBuilder logBuilder = new StringBuilder(); + ApiExecuteInfo apiExecuteInfo = new ApiExecuteInfo(); + ApiResDataInfo apiResDataInfo = new ApiResDataInfo(); + ArrayList> list = new ArrayList<>(); + try { + connection = PoolManager.getPooledConnection(datasource); + PreparedStatement statement = connection.prepareStatement(sql+pageSent); + //参数注入 + for (int i = 1; i <= jdbcParamValues.size(); i++) { + statement.setObject(i, jdbcParamValues.get(i - 1)); + } + logBuilder.append("sql:"+statement +"\n"); + if (isSelect == 1) { + ResultSet rs = statement.executeQuery(); + int columnCount = rs.getMetaData().getColumnCount(); + List columns = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + String columnName = rs.getMetaData().getColumnLabel(i); + columns.add(columnName); + } + while (rs.next()) { + HashMap jo = new HashMap<>(); + for (String columnName : columns) { + Object value = rs.getObject(columnName); + jo.put(columnName,value == null ? null :value.toString() ); + } + list.add(jo); + } + + //分页,计算总page + if(pageSize > 0){ + String countSql = PagerUtils.count(sql, DbType.MYSQL.getDb()); + PreparedStatement countStatement = connection.prepareStatement(countSql); + //参数注入 + for (int i = 1; i <= jdbcParamValues.size(); i++) { + countStatement.setObject(i, jdbcParamValues.get(i-1)); + } + ResultSet countRs = countStatement.executeQuery(); + if ((countRs.next())){ + int totalRecord = countRs.getInt(1); + int totalPage = (totalRecord + pageSize-1) / pageSize; + apiResDataInfo.setTotalPage(totalPage); + apiResDataInfo.setTotal(totalRecord); + apiResDataInfo.setPageSize(pageSize); + apiResDataInfo.setCurrentPageNum(pageNum); + } + } + + } else { + statement.executeUpdate(); + } + apiResDataInfo.setData(list); + } catch (Exception e){ + log.error("ERROR", "Error found: ", e); + logBuilder.append(e.getMessage()); + throw new DataApiException(logBuilder.toString()); + }finally { + connection.close(); + } + apiExecuteInfo.setLog(logBuilder.toString()); + apiExecuteInfo.setResList(apiResDataInfo); + return apiExecuteInfo; + } + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiDataSourceServiceImpl.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiDataSourceServiceImpl.java new file mode 100644 index 000000000..12b1c1ce5 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiDataSourceServiceImpl.java @@ -0,0 +1,96 @@ +package com.webank.wedatasphere.dss.data.api.server.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.data.api.server.dao.DataSourceMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.DataSource; +import com.webank.wedatasphere.dss.data.api.server.util.CryptoUtils; +import com.webank.wedatasphere.dss.data.api.server.service.ApiDataSourceService; +import com.webank.wedatasphere.dss.data.api.server.util.JdbcUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + + +@Service +public class ApiDataSourceServiceImpl extends ServiceImpl implements ApiDataSourceService { + @Resource + private DataSourceMapper dataSourceMapper; + private static final Logger LOGGER = LoggerFactory.getLogger(ApiDataSourceServiceImpl.class); + + @Override + public DataSource selectById(Integer datasourceId) { + + return dataSourceMapper.selectById(datasourceId); + } + + @Override + public List getAllConnections(Integer workspaceId, String type) { + DataSource dataSource = new DataSource(); + dataSource.setWorkspaceId(workspaceId); + dataSource.setType(type); + return dataSourceMapper.selectByTypeAndWorkspaceId(dataSource); + } + + @Override + public List getAvailableConns(List allConnections) { + + Connection connection = null; + List availableConns = new ArrayList(); + for (DataSource datasource : allConnections) { + try { + datasource.setPwd(CryptoUtils.string2Object(datasource.getPwd()).toString()); + connection = JdbcUtil.getConnection(datasource); + datasource.setPwd("***"); + availableConns.add(datasource); + } catch (Exception e) { + LOGGER.error(e.getMessage()); + } finally { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + LOGGER.info(e.getMessage()); + } + } + } + + } + return availableConns; + + } + + @Override + public void addDatasource(DataSource dataSource) { + dataSourceMapper.addDatasource(dataSource); + } + + @Override + public List listAllDatasources(DataSource dataSource) { + return dataSourceMapper.listAllDatasources(dataSource); + } + + @Override + public void deleteById(Integer id) { + dataSourceMapper.deleteById(id); + } + + @Override + public void editDatasource(DataSource dataSource) { + dataSourceMapper.editDatasource(dataSource); + } + + @Override + public boolean isDataSourceUsing(Integer id) { + if (dataSourceMapper.dataSourceUsingCount(id) > 0) { + return true; + } + return false; + } + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiManagerServiceImpl.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiManagerServiceImpl.java new file mode 100644 index 000000000..7cc549466 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiManagerServiceImpl.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.dss.data.api.server.service.impl; + +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.data.api.server.dao.ApiConfigMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiInfo; +import com.webank.wedatasphere.dss.data.api.server.service.ApiManagerService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class ApiManagerServiceImpl implements ApiManagerService { + @Autowired + private ApiConfigMapper apiConfigMapper; + + @Override + public ApiInfo getApiInfo(Long apiId){ + return apiConfigMapper.getApiInfo(apiId); + } + + @Override + public List getApiInfoList(Long workspaceId, String apiName, List totals, Integer pageNow, Integer pageSize){ + PageHelper.startPage(pageNow,pageSize,true); + // MYSQL LIKE % _: LIKE '%\_%', LIKE '%\%%' + if(apiName !=null) { + if ("_".equalsIgnoreCase(apiName.trim())) { + apiName = "\\_"; + } + if ("%".equalsIgnoreCase(apiName.trim())) { + apiName = "\\%"; + } + } + List apiInfoList = apiConfigMapper.getApiInfoList(workspaceId,apiName); + PageInfo pageInfo = new PageInfo<>(apiInfoList); + totals.add(pageInfo.getTotal()); + + return apiInfoList; + } + + @Override + public List getOnlineApiInfoList(Long workspaceId, String apiName, List totals, Integer pageNow, Integer pageSize){ + PageHelper.startPage(pageNow,pageSize,true); + // MYSQL LIKE % _: LIKE '%\_%', LIKE '%\%%' + if(apiName !=null) { + if ("_".equalsIgnoreCase(apiName.trim())) { + apiName = "\\_"; + } + if ("%".equalsIgnoreCase(apiName.trim())) { + apiName = "\\%"; + } + } + List apiInfoList = apiConfigMapper.getOnlineApiInfoList(workspaceId,apiName); + PageInfo pageInfo = new PageInfo<>(apiInfoList); + totals.add(pageInfo.getTotal()); + + return apiInfoList; + } + + @Override + public void offlineApi(Long apiId){ + apiConfigMapper.offlineApi(apiId); + } + + @Override + public void onlineApi(Long apiId){ + apiConfigMapper.onlineApi(apiId); + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiMonitorServiceImpl.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiMonitorServiceImpl.java new file mode 100644 index 000000000..367c48507 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/service/impl/ApiMonitorServiceImpl.java @@ -0,0 +1,130 @@ +package com.webank.wedatasphere.dss.data.api.server.service.impl; + +import com.webank.wedatasphere.dss.data.api.server.dao.ApiCallMapper; +import com.webank.wedatasphere.dss.data.api.server.dao.ApiConfigMapper; +import com.webank.wedatasphere.dss.data.api.server.entity.request.CallMonitorResquest; +import com.webank.wedatasphere.dss.data.api.server.entity.request.SingleCallMonitorRequest; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiCallInfoByCnt; +import com.webank.wedatasphere.dss.data.api.server.entity.response.ApiCallInfoByFailRate; +import com.webank.wedatasphere.dss.data.api.server.entity.response.HourMonitorInfo; +import com.webank.wedatasphere.dss.data.api.server.service.ApiMonitorService; +import com.webank.wedatasphere.dss.data.api.server.util.TimeUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class ApiMonitorServiceImpl implements ApiMonitorService { + @Autowired + private ApiConfigMapper apiConfigMapper; + + @Autowired + private ApiCallMapper apiCallMapper; + + @Override + public Long getOnlineApiCnt(Long workspaceId) { + return apiConfigMapper.getOnlineApiCnt(workspaceId); + } + + @Override + public Long getOfflineApiCnt(Long workspaceId) { + return apiConfigMapper.getOfflineApiCnt(workspaceId); + } + + @Override + public Long getCallTotalCnt(CallMonitorResquest callMonitorResquest) { + return apiCallMapper.getCallTotalCnt(callMonitorResquest); + } + + @Override + public Long getCallTotalTime(CallMonitorResquest callMonitorResquest) { + return apiCallMapper.getCallTotalTime(callMonitorResquest); + } + + @Override + public List getCallListByCnt(CallMonitorResquest callMonitorResquest) { + return apiCallMapper.getCallListByCnt(callMonitorResquest); + } + + @Override + public List getCallListByFailRate(CallMonitorResquest callMonitorResquest) { + return apiCallMapper.getCallListByFailRate(callMonitorResquest); + } + + @Override + public List getCallCntForPast24H(Long workspaceId) throws Exception { + //整点小时的时间点集合 + ArrayList timeTagList = TimeUtil.getTimeLagForPast24H(); + //小时整点-次数(很可能数据不全,会有数据缺失) + List> oldLMap = apiCallMapper.getCallCntForPast24H(workspaceId); + Map oldMap =transforLMap2Map(oldLMap); + + List list = new ArrayList<>(); + for (String timeTag: timeTagList) { + list.add(new HourMonitorInfo(timeTag,oldMap.getOrDefault(timeTag,0))); + } + + return list; + } + + @Override + public List getCallTimeForSinleApi(SingleCallMonitorRequest singleCallMonitorRequest) throws Exception { + //整点小时的时间点集合 + ArrayList timeTagList = TimeUtil.getTimeLag(singleCallMonitorRequest.getStartTime(),singleCallMonitorRequest.getEndTime()); + //小时整点-次数(很可能数据不全,会有数据缺失) + List> oldLMap = apiCallMapper.getCallTimeForSinleApi(singleCallMonitorRequest); + Map oldMap =transforLMap2Map(oldLMap); + + List list = new ArrayList<>(); + for (String timeTag: timeTagList) { + list.add(new HourMonitorInfo(timeTag,oldMap.getOrDefault(timeTag,0))); + } + + return list; + } + + @Override + public List getCallCntForSinleApi(SingleCallMonitorRequest singleCallMonitorRequest) throws Exception { + //整点小时的时间点集合 + ArrayList timeTagList = TimeUtil.getTimeLag(singleCallMonitorRequest.getStartTime(),singleCallMonitorRequest.getEndTime()); + //小时整点-次数(很可能数据不全,会有数据缺失) + List> oldLMap = apiCallMapper.getCallCntForSinleApi(singleCallMonitorRequest); + Map oldMap =transforLMap2Map(oldLMap); + + List list = new ArrayList<>(); + for (String timeTag: timeTagList) { + list.add(new HourMonitorInfo(timeTag,oldMap.getOrDefault(timeTag,0))); + } + + return list; + } + + + /** + * 转换数据格式 + */ + private Map transforLMap2Map(List> lMap){ + Map newMap = new HashMap<>(); + + for (Map map: lMap) { + String key =null; + Object value =null; + for (Map.Entry maps: map.entrySet()) { + //System.out.println("key"+maps.getKey()); + //System.out.println("key"+maps.getValue()); + if("k".equals(maps.getKey())){ + key = String.valueOf(maps.getValue()); + } + if("v".equals(maps.getKey())){ + value = String.valueOf(maps.getValue()); + } + } + newMap.put(key,value); + } + return newMap ; + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/CommUtil.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/CommUtil.java new file mode 100644 index 000000000..7e402e29a --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/CommUtil.java @@ -0,0 +1,95 @@ +package com.webank.wedatasphere.dss.data.api.server.util; + +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +import java.util.ArrayList; +import java.util.List; + +public class CommUtil { + /** + * 将object转换成数组 + * @param object + * @return + */ + public static String[] objectToArray(Object object) { + String[] newArray = null; + if (object != null) { + List result = new ArrayList<>(); + if (object instanceof ArrayList) { + for (Object o : (List) object) { + result.add(o == null ? null : String.valueOf(o)); + } + } + newArray = new String[result.size()]; + for (int i = 0; i < result.size(); i++) { + newArray[i] = result.get(i); + } + } + return newArray; + } + + /** + * 拼接where句柄 + * @param requestFields + * @return + * @throws JSONException + */ + + public static String getWhereCause(String requestFields) throws JSONException { + JSONArray jsonArray = new JSONArray(requestFields); + StringBuilder whereCauseBuild = new StringBuilder(); + int index = 0; + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + String columnName = jsonObject.getString("name").trim(); + if("pageNum".equalsIgnoreCase(columnName)) continue; + if (index == 0) { + whereCauseBuild.append(" where "); + } + index++; + String columnNameFormat = String.format("%s%s%s","`",columnName,"`"); + String compareType = jsonObject.getString("compare").replaceAll("<","<"); + String whereCause = String.format("%s %s #{%s}", columnNameFormat, compareType, columnName); + whereCauseBuild.append(whereCause).append(" and "); + + } + String whereStr = whereCauseBuild.toString(); + int whereStrSize = whereStr.length(); + if(whereStr.endsWith(" and ")){ + whereStr = whereStr.substring(0,whereStrSize-5); + + } + return whereStr; + } + + /** + * 拼接 order by 句柄 + * @param orderFields + * @return + * @throws JSONException + */ + public static String getOrderCause(String orderFields) throws JSONException { + JSONArray jsonArray = new JSONArray(orderFields); + StringBuilder orderCauseBuild = new StringBuilder(); + for (int i = 0; i < jsonArray.length(); i++) { + if (i == 0) { + orderCauseBuild.append(" order by "); + } + JSONObject jsonObject = jsonArray.getJSONObject(i); + String columnName = jsonObject.getString("name").trim(); + columnName = String.format("%s%s%s","`",columnName,"`"); + String orderType = jsonObject.getString("type"); + String orderCause = String.format("%s %s", columnName, orderType); + System.out.println(orderCause); + if (i < jsonArray.length() - 1) { + orderCauseBuild.append(orderCause).append(","); + } else { + orderCauseBuild.append(orderCause); + } + } + return orderCauseBuild.toString(); + } + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/CryptoUtils.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/CryptoUtils.java new file mode 100644 index 000000000..0dc82b1c6 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/CryptoUtils.java @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.dss.data.api.server.util; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.security.MessageDigest; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang.StringUtils; + +public class CryptoUtils { + private CryptoUtils() { + } + + public static String object2String(Serializable o) { + try { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(o); + oos.flush(); + oos.close(); + bos.close(); + return new String((new Base64()).encode(bos.toByteArray())); + } catch (Exception var3) { + throw new RuntimeException(var3); + } + } + + public static Object string2Object(String str) { + try { + ByteArrayInputStream bis = new ByteArrayInputStream((new Base64()).decode(str.getBytes())); + ObjectInputStream ois = new ObjectInputStream(bis); + Object o = ois.readObject(); + bis.close(); + ois.close(); + return o; + } catch (Exception var4) { + throw new RuntimeException(var4); + } + } + + public static String md5(String source, String salt, int iterator) { + StringBuilder token = new StringBuilder(); + + try { + MessageDigest digest = MessageDigest.getInstance("md5"); + if (StringUtils.isNotEmpty(salt)) { + digest.update(salt.getBytes("UTF-8")); + } + + byte[] result = digest.digest(source.getBytes()); + + for(int i = 0; i < iterator - 1; ++i) { + digest.reset(); + result = digest.digest(result); + } + + byte[] var12 = result; + int var7 = result.length; + + for(int var8 = 0; var8 < var7; ++var8) { + byte aResult = var12[var8]; + int temp = aResult & 255; + if (temp <= 15) { + token.append("0"); + } + + token.append(Integer.toHexString(temp)); + } + + return token.toString(); + } catch (Exception var11) { + throw new RuntimeException(var11.getMessage()); + } + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/DateJsonDeserializer.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/DateJsonDeserializer.java new file mode 100644 index 000000000..bb1d044ba --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/DateJsonDeserializer.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.data.api.server.util; + +import org.codehaus.jackson.JsonParser; +import org.codehaus.jackson.JsonProcessingException; +import org.codehaus.jackson.map.DeserializationContext; +import org.codehaus.jackson.map.JsonDeserializer; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class DateJsonDeserializer extends JsonDeserializer +{ + public static final SimpleDateFormat format=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + @Override + public Date deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException, JsonProcessingException + { + try + { + return format.parse(jsonParser.getText()); + } + catch(Exception e) + { + System.out.println(e.getMessage()); + throw new RuntimeException(e); + } + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/DateJsonSerializer.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/DateJsonSerializer.java new file mode 100644 index 000000000..1548167cd --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/DateJsonSerializer.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.data.api.server.util; + +import org.codehaus.jackson.JsonGenerator; +import org.codehaus.jackson.JsonProcessingException; +import org.codehaus.jackson.map.JsonSerializer; +import org.codehaus.jackson.map.SerializerProvider; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class DateJsonSerializer extends JsonSerializer +{ + public static final SimpleDateFormat format=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + @Override + public void serialize(Date date, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException, JsonProcessingException + { + jsonGenerator.writeString(format.format(date)); + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/JdbcUtil.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/JdbcUtil.java new file mode 100644 index 000000000..89aa9b679 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/JdbcUtil.java @@ -0,0 +1,152 @@ +package com.webank.wedatasphere.dss.data.api.server.util; + +import com.alibaba.druid.util.JdbcConstants; +import com.webank.wedatasphere.dss.data.api.server.entity.DataSource; +import lombok.extern.slf4j.Slf4j; + +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +@Slf4j +public class JdbcUtil { + + public static ResultSet query(String sql, Connection connection) throws SQLException { + PreparedStatement preparedStatement = connection.prepareStatement(sql); + ResultSet resultSet = preparedStatement.executeQuery(); + return resultSet; + } + + public static Connection getConnection(DataSource ds) throws SQLException, ClassNotFoundException { + String url = ds.getUrl(); + switch (ds.getType()) { + case JdbcConstants.MYSQL: + Class.forName(JdbcConstants.MYSQL_DRIVER); + break; + case JdbcConstants.POSTGRESQL: + Class.forName(JdbcConstants.POSTGRESQL_DRIVER); + break; + case JdbcConstants.HIVE: + Class.forName(JdbcConstants.HIVE_DRIVER); + break; + case JdbcConstants.SQL_SERVER: + Class.forName(JdbcConstants.SQL_SERVER_DRIVER_SQLJDBC4); + break; + case JdbcConstants.CLICKHOUSE: + Class.forName(JdbcConstants.CLICKHOUSE_DRIVER); + break; + case JdbcConstants.KYLIN: + Class.forName(JdbcConstants.KYLIN_DRIVER); + break; + case JdbcConstants.ORACLE: + Class.forName(JdbcConstants.ORACLE_DRIVER); + break; + default: + break; + } + + Connection connection = DriverManager.getConnection(url, ds.getUsername(), ds.getPwd()); + log.info("获取连接成功"); + return connection; + } + + /** + * 查询库中所有表 + * @param conn + * @param type + * @return + */ + public static List getAllTables(Connection conn, String type) { + List list = new ArrayList<>(); + PreparedStatement pst = null; + try { + String sql; + switch (type) { + case "MYSQL": + case "HIVE": + sql = "show tables"; + break; + case "POSTGRESQL": + sql = "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name"; + break; + // TODO + case "SQLSERVER": + sql = "select * from sys.tables"; + break; + default: + sql = "show tables"; + + } + + pst = conn.prepareStatement(sql); + ResultSet resultSet = pst.executeQuery(); + + while (resultSet.next()) { + String s = resultSet.getString(1); + list.add(s); + } + return list; + } catch (Exception e) { + log.error(e.getMessage(), e); + return null; + } finally { + try { + if (pst != null) + pst.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + /** + * 查询表所有字段 + * @param conn + * @param type + * @param table + * @return + */ + public static List getRDBMSColumnProperties(Connection conn, String type, String table) { + List list = new ArrayList<>(); + + ResultSet resultSet = null; + try { + String sql; + switch (type) { + case "POSTGRESQL": + sql = "select * from \"" + table + "\" where 1=2"; + break; + default: + sql = "show full columns from " + table; + } + log.info(sql); + + resultSet = conn.prepareStatement(sql).executeQuery(); + while (resultSet.next()) { + HashMap colProp = new HashMap<>(); + colProp.put("Comment",resultSet.getString("Comment")); + colProp.put("fieldType",resultSet.getString("Type")); + colProp.put("columnName",resultSet.getString("Field")); + + list.add(colProp); + } + return list; + } catch (Exception e) { + log.error(e.getMessage(), e); + return null; + } finally { + try { + if (resultSet != null) + resultSet.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + +} \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/PoolManager.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/PoolManager.java new file mode 100644 index 000000000..15d7259f0 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/PoolManager.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.dss.data.api.server.util; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.DruidPooledConnection; +import com.webank.wedatasphere.dss.data.api.server.entity.DataSource; +import lombok.extern.slf4j.Slf4j; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + + +@Slf4j +public class PoolManager { + + private static Lock lock = new ReentrantLock(); + + private static Lock deleteLock = new ReentrantLock(); + + //所有数据源的连接池存在map里 + static Map map = new HashMap<>(); + + public static DruidDataSource getJdbcConnectionPool(DataSource ds) { + if (map.containsKey(ds.getDatasourceId())) { + return map.get(ds.getDatasourceId()); + } else { + lock.lock(); + try { + log.info(Thread.currentThread().getName() + "获取锁"); + if (!map.containsKey(ds.getDatasourceId())) { + DruidDataSource druidDataSource = new DruidDataSource(); + druidDataSource.setName(ds.getName()); + druidDataSource.setUrl(ds.getUrl()); + druidDataSource.setUsername(ds.getUsername()); + druidDataSource.setPassword(ds.getPwd()); + druidDataSource.setDriverClassName(ds.getClassName()); + druidDataSource.setConnectionErrorRetryAttempts(3); //失败后重连次数 + druidDataSource.setBreakAfterAcquireFailure(true); + + map.put(ds.getDatasourceId(), druidDataSource); + log.info("创建Druid连接池成功:{}", ds.getName()); + } + return map.get(ds.getDatasourceId()); + } catch (Exception e) { + return null; + } finally { + lock.unlock(); + } + } + } + + //删除数据库连接池 + public static void removeJdbcConnectionPool(Integer id) { + deleteLock.lock(); + try { + DruidDataSource druidDataSource = map.get(id); + if (druidDataSource != null) { + druidDataSource.close(); + map.remove(id); + } + } catch (Exception e) { + log.error(e.toString()); + } finally { + deleteLock.unlock(); + } + + } + + public static DruidPooledConnection getPooledConnection(DataSource ds) throws SQLException { + ds.setPwd(CryptoUtils.string2Object(ds.getPwd()).toString()); + DruidDataSource pool = PoolManager.getJdbcConnectionPool(ds); + DruidPooledConnection connection = pool.getConnection(); + return connection; + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/RestfulUtils.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/RestfulUtils.java new file mode 100644 index 000000000..d64c6214a --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/RestfulUtils.java @@ -0,0 +1,49 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.api.server.util; + +import org.apache.linkis.server.Message; +import org.apache.commons.math3.util.Pair; + +import javax.ws.rs.core.Response; +import java.util.Arrays; + +public class RestfulUtils { + +// public static Response dealError(String reason){ +// Message message = Message.error(reason); +// return Message.messageToResponse(message); +// } +// +// public static Response dealOk(String msg){ +// Message message = Message.ok(msg); +// return Message.messageToResponse(message); +// } +// +// +// +// @SafeVarargs +// public static Response dealOk(String msg, Pair... data){ +// Message message = Message.ok(msg); +// Arrays.stream(data).forEach(p -> message.data(p.getKey(), p.getValue())); +// return Message.messageToResponse(message); +// } + + +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/SqlEngineUtil.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/SqlEngineUtil.java new file mode 100644 index 000000000..89188190d --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/SqlEngineUtil.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.data.api.server.util; +import com.webank.wedatasphere.dss.orange.DynamicSqlEngine; + + +public class SqlEngineUtil { + + static DynamicSqlEngine engine = new DynamicSqlEngine(); + + public static DynamicSqlEngine getEngine() { + return engine; + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/TimeUtil.java b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/TimeUtil.java new file mode 100644 index 000000000..8f11fe034 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/java/com/webank/wedatasphere/dss/data/api/server/util/TimeUtil.java @@ -0,0 +1,101 @@ +package com.webank.wedatasphere.dss.data.api.server.util; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; + +public class TimeUtil { + /** + * 时间段的小时数 + */ + public static long getHourCnt(String startTime,String endTime) throws Exception { + return dateDiff(startTime,endTime,"yyyy-MM-dd HH:mm:ss"); + } + + private static long dateDiff(String startTime, String endTime, String format) throws Exception { + SimpleDateFormat sd = new SimpleDateFormat(format); + //long nd = 1000*24*60*60; //一天的毫秒数 + long nh = 1000*60*60; //一小时的毫秒数 + //long nm = 1000*60; //一分钟的毫秒数 + //long ns = 1000; //一秒钟的毫秒数 + long diff; + //获得两个时间的毫秒时间差异 + diff = sd.parse(endTime).getTime() - sd.parse(startTime).getTime(); + //long day = diff/nd;//计算差多少天 + //long hour = diff % nd / nh;//计算差多少小时 + //long min = diff%nd%nh/nm;//计算差多少分钟 + //long sec = diff%nd%nh%nm/ns;//计算差多少秒//输出结果 + //System.out.println("时间相差:"+day+"天"+hour+"小时"+min+"分钟"+sec+"秒。"); + + long hour = diff / nh;//计算差多少小时 + return hour ; + } + + public static ArrayList getTimeLagForPast24H() throws Exception { + //创建集合存储所有时间点 + ArrayList list = new ArrayList (); + + String format = "yyyy-MM-dd HH:mm:ss"; + SimpleDateFormat sdf = new SimpleDateFormat(format); + + Calendar ca = Calendar.getInstance(); + ca.set(Calendar.MINUTE, 0); + ca.set(Calendar.SECOND, 0); + + SimpleDateFormat sdfHour = new SimpleDateFormat("yyyy-MM-dd HH:00"); + for(int i= 0; i< 24; i++){ + list.add(sdfHour.format(ca.getTime())); + ca.add(Calendar.HOUR,-1); + } + + Collections.reverse(list); + return list; + } + + //获取时间段内小时整点 + public static ArrayList getTimeLag(String startTime, String endTime) throws Exception { + //创建集合存储所有时间点 + ArrayList list = new ArrayList (); + + String format = "yyyy-MM-dd HH:mm:ss"; + SimpleDateFormat sdf = new SimpleDateFormat(format); + Date dBegin = sdf.parse(startTime); + Date dEnd = sdf.parse(endTime); + + Calendar ca = Calendar.getInstance(); + ca.setTime(dEnd); + ca.set(Calendar.MINUTE, 0); + ca.set(Calendar.SECOND, 0); + Date dEndHour = ca.getTime(); + + SimpleDateFormat sdfHour = new SimpleDateFormat("yyyy-MM-dd HH:00"); + while(dEndHour.compareTo(dBegin) >=0){ + list.add(sdfHour.format(dEndHour)); + + ca.add(Calendar.HOUR,-1); + dEndHour =ca.getTime(); + } + + Collections.reverse(list); + return list; + } + + /** + * 获取当前时间的整点小时时间 + */ + public static String getCurrHourTime( ){ + Calendar ca = Calendar.getInstance(); + ca.set(Calendar.MINUTE, 0); + ca.set(Calendar.SECOND, 0); + Date date = ca.getTime(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:00"); + return sdf.format(date); + } + + public static void main(String[] args) throws Exception { + //System.out.println(getHourCnt("2021-07-29 00:00:00","2021-08-05 00:00:00")); + System.out.println(getTimeLag("2021-08-11 00:10:00","2021-08-12 00:20:00")); + } +} diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/application-dss.yml b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/application-dss.yml new file mode 100644 index 000000000..b0473838d --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/application-dss.yml @@ -0,0 +1,23 @@ + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + #instance: + #prefer-ip-address: true + #instance-id: ${spring.cloud.client.ip-address}:${server.port} + #metadata-map: + #test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss-data-api-server.properties b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss-data-api-server.properties new file mode 100644 index 000000000..fe386097e --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss-data-api-server.properties @@ -0,0 +1,39 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9208 +spring.spring.application.name=dss-data-api-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.api.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/api/server/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.api.server.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.api.server.dao + +#wds.linkis.gateway.ip=127.0.0.1 +#wds.linkis.gateway.port=9001 +#wds.linkis.gateway.url=http://127.0.0.1:9001/ diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss.properties b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss.properties new file mode 100644 index 000000000..50a90b064 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss.properties @@ -0,0 +1,25 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +wds.linkis.gateway.ip=127.0.0.1 +wds.linkis.gateway.port=9001 +wds.linkis.gateway.url=http://127.0.0.1:9001/ + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/dss_test?characterEncoding=UTF-8 +wds.linkis.server.mybatis.datasource.username= +wds.linkis.server.mybatis.datasource.password= \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/log4j.properties b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/log4j.properties new file mode 100644 index 000000000..3f4c83b72 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/log4j.properties @@ -0,0 +1,38 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/dss-apiservice-server.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/log4j2.xml b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/log4j2.xml new file mode 100644 index 000000000..2061dd726 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/token.properties b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/token.properties new file mode 100644 index 000000000..a52a8983d --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/token.properties @@ -0,0 +1,19 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +${userName}=${password} \ No newline at end of file diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/scala/com/webank/wedatasphere/dss/data/api/server/DSSDataApiApplication.scala b/dss-apps/dss-data-api/dss-data-api-server/src/main/scala/com/webank/wedatasphere/dss/data/api/server/DSSDataApiApplication.scala new file mode 100644 index 000000000..4aa992e18 --- /dev/null +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/scala/com/webank/wedatasphere/dss/data/api/server/DSSDataApiApplication.scala @@ -0,0 +1,41 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.api.server + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + + +object DSSDataApiApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-apps/dss-data-api/pom.xml b/dss-apps/dss-data-api/pom.xml new file mode 100644 index 000000000..120c8e961 --- /dev/null +++ b/dss-apps/dss-data-api/pom.xml @@ -0,0 +1,20 @@ + + + + dss + com.webank.wedatasphere.dss + ../../pom.xml + 1.1.1 + + 4.0.0 + dss-data-api + pom + + dss-data-api-server + dss-api-sql-template + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml b/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml new file mode 100644 index 000000000..293485146 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml @@ -0,0 +1,223 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-data-asset-server + + + 2.1.0 + + + + com.webank.wedatasphere.dss + dss-data-governance-common + ${dss.version} + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + org.springframework.cloud + spring-cloud-netflix + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + javax.ws.rs + javax.ws.rs-api + + + + + org.apache.commons + commons-math3 + provided + + + xstream + com.thoughtworks.xstream + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + ${spring.cloud.version} + provided + + + logback-classic + ch.qos.logback + + + log4j-to-slf4j + org.apache.logging.log4j + + + gson + com.google.code.gson + + + jsr311-api + javax.ws.rs + + + xstream + com.thoughtworks.xstream + + + commons-math + org.apache.commons + + + jackson-core + com.fasterxml.jackson.core + + + spring-boot-autoconfigure + org.springframework.boot + + + spring-boot-starter-aop + org.springframework.boot + + + spring-boot-starter + org.springframework.boot + + + spring-boot-starter-cache + org.springframework.boot + + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + org.apache.atlas + atlas-client-v2 + ${atlas.version} + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + + + org.projectlombok + lombok + 1.18.16 + compile + + + + com.alibaba + druid + 1.1.9 + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/assembly/distribution.xml b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..46d83d32f --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/assembly/distribution.xml @@ -0,0 +1,46 @@ + + + + dss-data-asset-server + + dir + + true + dss-data-asset-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/MetaInfoMapper.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/MetaInfoMapper.java new file mode 100644 index 000000000..bb2b0ff5a --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/MetaInfoMapper.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.data.asset.dao; + +import com.webank.wedatasphere.dss.data.asset.entity.HivePartInfo; +import com.webank.wedatasphere.dss.data.asset.entity.HiveStorageInfo; +import org.apache.ibatis.annotations.*; + +import java.sql.SQLException; +import java.util.List; + +@Mapper +public interface MetaInfoMapper { + Long getTableStorage() throws SQLException; + List getTop10Table() throws SQLException; + int getTableInfo(@Param("dbName") String dbName,@Param("tableName") String tableName,@Param("isPartTable") Boolean isPartTable) throws SQLException; + + List getPartInfo(@Param("dbName") String dbName, @Param("tableName") String tableName) throws SQLException; + + + +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/WorkspaceInfoMapper.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/WorkspaceInfoMapper.java new file mode 100644 index 000000000..69091da49 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/WorkspaceInfoMapper.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.data.asset.dao; + +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + + +import java.util.List; + +@Mapper +public interface WorkspaceInfoMapper { + + @Select("select distinct username from dss_workspace_user_role where workspace_id= #{workspaceId} and username like #{search} ") + List getWorkspaceUsersName (@Param("workspaceId") int workspaceId,@Param("search") String search); + +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/impl/MetaInfoMapperImpl.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/impl/MetaInfoMapperImpl.java new file mode 100644 index 000000000..72989801d --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/impl/MetaInfoMapperImpl.java @@ -0,0 +1,174 @@ +package com.webank.wedatasphere.dss.data.asset.dao.impl; + +import com.webank.wedatasphere.dss.data.asset.dao.MetaInfoMapper; +import com.webank.wedatasphere.dss.data.asset.entity.HivePartInfo; +import com.webank.wedatasphere.dss.data.asset.entity.HiveStorageInfo; +import com.webank.wedatasphere.dss.data.common.exception.DAOException; +import com.webank.wedatasphere.dss.data.common.utils.DataSourceUtil; +import com.webank.wedatasphere.dss.data.common.utils.DateUtil; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class MetaInfoMapperImpl implements MetaInfoMapper { + @Override + public Long getTableStorage() throws SQLException { + DataSource dataSource = DataSourceUtil.getDataSource(); + + Connection con =dataSource.getConnection(); + long num=0; + PreparedStatement ps=null; + ResultSet rs=null; + try { + String sql="select SUM(PARAM_VALUE) from TABLE_PARAMS WHERE PARAM_KEY='totalSize'"; + ps=con.prepareStatement(sql); + rs=ps.executeQuery(); + while (rs.next()){ + num =rs.getLong(1); + } + String sql2 ="select SUM(PARAM_VALUE) from PARTITION_PARAMS WHERE PARAM_KEY='totalSize'"; + ps=con.prepareStatement(sql2); + rs=ps.executeQuery(); + while (rs.next()){ + num= num + rs.getLong(1); + } + + } catch (DAOException | SQLException e){ + throw new DAOException(e.getMessage(),e); + } + finally { + con.close(); + } + + return num; + } + + @Override + public List getTop10Table() throws SQLException{ + DataSource dataSource = DataSourceUtil.getDataSource(); + Connection con =dataSource.getConnection(); + PreparedStatement ps=null; + ResultSet rs=null; + List hiveStorageInfos = new ArrayList<>(); + try { + String sql="SELECT DBS.NAME ,TBLS.TBL_NAME,CAST(TABLE_PARAMS.PARAM_VALUE AS UNSIGNED) AS totalSize from DBS, TBLS,TABLE_PARAMS where TBLS.TBL_ID=TABLE_PARAMS.TBL_ID AND TBLS.DB_ID=DBS.DB_ID AND TABLE_PARAMS.PARAM_KEY='totalSize' order by totalSize DESC limit 10"; + ps=con.prepareStatement(sql); + rs=ps.executeQuery(); + while (rs.next()){ + HiveStorageInfo tableinfo=new HiveStorageInfo(); + tableinfo.setTableName(rs.getString(1)+"."+rs.getString(2)); + tableinfo.setStorage(rs.getLong(3)); + hiveStorageInfos.add(tableinfo); + } + String sql2="SELECT DBS.NAME ,TBLS.TBL_NAME,SUM(CAST(PARTITION_PARAMS.PARAM_VALUE AS UNSIGNED)) AS totalSize from DBS,TBLS,PARTITIONS ,PARTITION_PARAMS where DBS.DB_ID=TBLS.DB_ID AND TBLS.TBL_ID=PARTITIONS.TBL_ID AND PARTITIONS.PART_ID =PARTITION_PARAMS.PART_ID AND PARTITION_PARAMS.PARAM_KEY='totalSize' group by TBLS.TBL_NAME order by totalSize desc limit 10"; + ps=con.prepareStatement(sql2); + rs=ps.executeQuery(); + while (rs.next()){ + HiveStorageInfo tableinfo=new HiveStorageInfo(); + tableinfo.setTableName(rs.getString(1)+"."+rs.getString(2)); + tableinfo.setStorage(rs.getLong(3)); + hiveStorageInfos.add(tableinfo); + } + /** + * 特别注意LONG类型相减超出INT范围 + * System.out.println((int) (4401131805L -1796673800L)) + * System.out.println(Long.parseLong("4401131805")-Long.parseLong("1796673800")) + */ + Collections.sort(hiveStorageInfos, new Comparator() { + @Override + public int compare(HiveStorageInfo o1, HiveStorageInfo o2) { + //return (int) (Long.valueOf(o2.getStorage())-Long.valueOf(o1.getStorage())) + if(o2.getStorage() > o1.getStorage()){ + return 1; + } + else if(o2.getStorage() < o1.getStorage()){ + return -1; + } + else{ + return 0; + } + } + }); + } catch (DAOException | SQLException e){ + throw new DAOException(e.getMessage(),e); + } + finally { + con.close(); + } + return hiveStorageInfos.subList(0,10); + } + + @Override + public int getTableInfo(String dbName, String tableName, Boolean isPartTable) throws SQLException { + DataSource dataSource = DataSourceUtil.getDataSource(); + Connection con =dataSource.getConnection(); + PreparedStatement ps=null; + ResultSet rs=null; + int res = 0; + try { + String sql=null; + if(isPartTable==false){ + sql="select TABLE_PARAMS.PARAM_VALUE as totalSize from DBS, TBLS,TABLE_PARAMS where TBLS.TBL_ID=TABLE_PARAMS.TBL_ID AND TBLS.DB_ID=DBS.DB_ID AND TABLE_PARAMS.PARAM_KEY='totalSize' AND DBS.NAME="+"'"+dbName+"' AND TBLS.TBL_NAME="+"'"+tableName+"'"; + } + else { + + sql="select SUM(PARTITION_PARAMS.PARAM_VALUE) as totalSize from DBS,TBLS,PARTITIONS ,PARTITION_PARAMS where DBS.DB_ID=TBLS.DB_ID AND TBLS.TBL_ID=PARTITIONS.TBL_ID AND PARTITIONS.PART_ID =PARTITION_PARAMS.PART_ID AND PARTITION_PARAMS.PARAM_KEY='totalSize' AND DBS.NAME="+"'"+dbName +"' AND TBLS.TBL_NAME="+"'"+tableName+"' group by TBLS.TBL_NAME"; + } + ps=con.prepareStatement(sql); + rs=ps.executeQuery(); + while (rs.next()){ + res=rs.getInt(1); + } + + } catch (DAOException | SQLException e){ + throw new DAOException(e.getMessage(),e); + } + finally { + con.close(); + } + return res; + } + + @Override + public List getPartInfo(String dbName, String tableName)throws SQLException { + DataSource dataSource = DataSourceUtil.getDataSource(); + Connection con =dataSource.getConnection(); + PreparedStatement ps=null; + ResultSet rs=null; + List hivePartInfos = new ArrayList<>(); + try { + String sql="select b.PART_NAME,b.CREATE_TIME,MAX(CASE c.PARAM_KEY WHEN 'transient_lastDdlTime' THEN c.PARAM_VALUE ELSE null END) transient_lastDdlTime ,MAX(CASE c.PARAM_KEY WHEN 'numRows' THEN c.PARAM_VALUE ELSE null END) numRows,MAX(CASE c.PARAM_KEY WHEN 'totalSize' THEN c.PARAM_VALUE ELSE null END) totalSize from TBLS a,PARTITIONS b,PARTITION_PARAMS c,DBS d where a.TBL_NAME="+"'"+tableName+"'"+"AND d.NAME="+"'"+dbName+"'" +"AND a.TBL_ID=b.TBL_ID AND a.DB_ID=d.DB_ID AND b.PART_ID=c.PART_ID GROUP BY c.PART_ID"; + ps=con.prepareStatement(sql); + rs=ps.executeQuery(); + while (rs.next()){ + HivePartInfo part =new HivePartInfo(); + part.setPartName(rs.getString(1)); + Long lastAccessTime = Long.valueOf(rs.getInt(3)); + if(lastAccessTime !=null && lastAccessTime !=0L) { + part.setLastAccessTime(DateUtil.unixToTimeStr(lastAccessTime * 1000)); + } + Long createTime = Long.valueOf(rs.getInt(2)); + if(createTime !=null && createTime !=0L) { + part.setCreateTime(DateUtil.unixToTimeStr(createTime * 1000)); + } + part.setReordCnt(rs.getInt(4)); + part.setStore(rs.getInt(5)); + hivePartInfos.add(part); + } + + } catch (DAOException | SQLException e){ + throw new DAOException(e.getMessage(),e); + } + finally { + con.close(); + } + return hivePartInfos; + } +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/impl/WorkspaceInfoMapper.xml b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/impl/WorkspaceInfoMapper.xml new file mode 100644 index 000000000..8dcfde4e3 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/dao/impl/WorkspaceInfoMapper.xml @@ -0,0 +1,24 @@ + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HivePartInfo.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HivePartInfo.java new file mode 100644 index 000000000..79ee1dafc --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HivePartInfo.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.data.asset.entity; + +import lombok.Data; + +@Data +public class HivePartInfo { + private String partName; + private int reordCnt; + private int store; + private String createTime; + private String lastAccessTime; + +} \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveStorageInfo.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveStorageInfo.java new file mode 100644 index 000000000..75d60bbb5 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveStorageInfo.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.data.asset.entity; + +import lombok.Data; + +@Data +public class HiveStorageInfo { + private String tableName; + private Long storage; + private String guid; +} \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblClassificationInfo.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblClassificationInfo.java new file mode 100644 index 000000000..7977271e8 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblClassificationInfo.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.asset.entity; + +import lombok.Data; + +import java.util.List; + +@Data +public class HiveTblClassificationInfo { + private List oldClassifications; + private List newClassifications; +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblDetailInfo.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblDetailInfo.java new file mode 100644 index 000000000..1204e2e9b --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblDetailInfo.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.dss.data.asset.entity; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.Set; + +@Data +public class HiveTblDetailInfo implements Serializable { + private HiveTblBasicInfo basic; + private List columns; + private List partitionKeys; + private List classifications; + + @Data + public static class HiveTblBasicInfo extends HiveTblSimpleInfo { + private String store; //存储量 + private Boolean isParTbl; //是否分区表 + private String tableType; //Hive表类型 tableType: EXTERNAL_TABLE, MANAGED_TABLE + private String location; //Hive表存储路径 + } + + @Data + public static class HiveColumnInfo { + private String name; + private String type; + private String guid; + private String comment; + } + + @Data + @AllArgsConstructor + public static class HiveClassificationInfo { + private String typeName; + private Set superTypeNames; + private Set subTypeNames; + } +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblLabelInfo.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblLabelInfo.java new file mode 100644 index 000000000..5a4be4a35 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblLabelInfo.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.data.asset.entity; + +import lombok.Data; + +import java.util.Set; + +@Data +public class HiveTblLabelInfo { + private Set labels; +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblSimpleInfo.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblSimpleInfo.java new file mode 100644 index 000000000..9ae3f3373 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblSimpleInfo.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.data.asset.entity; + +import lombok.Data; + +import java.util.List; +import java.util.Set; + +@Data +public class HiveTblSimpleInfo { + private String guid; + private String name; + private String dbName; + private String qualifiedName; + private List columns; + private String createTime; + private String owner; + private String comment; + private Set labels; + private List classifications; +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/restful/AssetRestful.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/restful/AssetRestful.java new file mode 100644 index 000000000..44c7dbeba --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/restful/AssetRestful.java @@ -0,0 +1,91 @@ +package com.webank.wedatasphere.dss.data.asset.restful; + +import com.webank.wedatasphere.dss.data.asset.entity.HiveTblLabelInfo; +import com.webank.wedatasphere.dss.data.asset.service.AssetService; +import com.webank.wedatasphere.dss.data.asset.service.WorkspaceInfoService; +import org.apache.linkis.server.Message; +import lombok.AllArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping(path = "/dss/data/governance/asset", produces = {"application/json"}) +@AllArgsConstructor +public class AssetRestful { + private static final Logger logger = LoggerFactory.getLogger(AssetRestful.class); + + private AssetService assetService; + private WorkspaceInfoService workspaceInfoService; + + /** + * 获取数据资产概要:hivedb数、hivetable数据、总存储量 + */ + @RequestMapping(method = RequestMethod.GET, path ="/hiveSummary") + public Message getHiveSummary() throws Exception { + return Message.ok().data("result", assetService.getHiveSummary()); + } + + /** + * 修改单个表或单个列注释 + */ + @RequestMapping(method = RequestMethod.PUT, path ="/comment/{guid}") + public Message modifyComment(@PathVariable String guid, @RequestParam String comment) throws Exception { + comment="\""+comment+"\""; + assetService.modifyComment(guid,comment); + return Message.ok().data("result","修改成功"); + } + + /** + * 批量修改多个个表或列注释 + */ + @RequestMapping(method = RequestMethod.PUT, path ="/comment/bulk") + public Message modifyComment(@RequestBody Map commentMap) throws Exception { + for (Map.Entry stringStringEntry : commentMap.entrySet()) { + stringStringEntry.setValue("\""+stringStringEntry.getValue()+"\""); + } + assetService.bulkModifyComment(commentMap); + + return Message.ok().data("result","修改成功"); + } + + + /** + * 设置单个表或单个列的标签 + */ + @RequestMapping(method = RequestMethod.POST, path ="/label/{guid}") + public Message setLabels(@PathVariable String guid, @RequestBody HiveTblLabelInfo hiveTblLabelInfo) throws Exception { + assetService.setLabels(guid,hiveTblLabelInfo.getLabels()); + + return Message.ok().data("result","设置成功"); + } + + /** + * 删除单个表或单个列的标签,linkis-gateway不支持DELETE方法 + */ + @RequestMapping(method = RequestMethod.PUT, path ="/label/{guid}") + public Message removeLabels(@PathVariable String guid, @RequestBody HiveTblLabelInfo hiveTblLabelInfo) throws Exception { + assetService.removeLabels(guid,hiveTblLabelInfo.getLabels()); + + return Message.ok().data("result","删除成功"); + } + + /** + * 获取工作空间下所有用户名 + */ + @RequestMapping(method = RequestMethod.GET, path ="getWorkspaceUsers/{workspaceId}/{search}") + public Message getWorkspaceUsers(@PathVariable int workspaceId,@PathVariable String search) throws Exception{ + String searchs="%"+search+"%"; + List workspaceUsers = workspaceInfoService.getWorkspaceUsers(workspaceId,searchs); + return Message.ok().data("result",workspaceUsers); + + } +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/restful/AssetTblRestful.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/restful/AssetTblRestful.java new file mode 100644 index 000000000..e74039753 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/restful/AssetTblRestful.java @@ -0,0 +1,187 @@ +package com.webank.wedatasphere.dss.data.asset.restful; + +import com.webank.wedatasphere.dss.data.asset.entity.HivePartInfo; +import com.webank.wedatasphere.dss.data.asset.entity.HiveStorageInfo; +import com.webank.wedatasphere.dss.data.asset.entity.HiveTblClassificationInfo; +import com.webank.wedatasphere.dss.data.asset.entity.HiveTblSimpleInfo; +import com.webank.wedatasphere.dss.data.asset.service.AssetService; +import com.webank.wedatasphere.dss.data.asset.service.WorkspaceInfoService; +import org.apache.linkis.server.Message; +import lombok.AllArgsConstructor; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@RestController +@RequestMapping(path = "/dss/data/governance/asset/hiveTbl", produces = {"application/json"}) +@AllArgsConstructor +public class AssetTblRestful { + private static final Logger logger = LoggerFactory.getLogger(AssetTblRestful.class); + + private static final String DEFAULT_DIRECTION = "BOTH"; + private static final String DEFAULT_DEPTH = "3"; + private static final String DEFAULT_LIMIT = "10"; + private static final String DEFAULT_OFFSET = "0"; + + private AssetService assetService; + private WorkspaceInfoService workspaceInfoService; + + /** + * 获取存储量前10的表信息 + */ + @RequestMapping(method = RequestMethod.GET, path ="/topStorage") + public Message getTop10Storage() throws Exception{ + List top10Table = assetService.getTop10Table(); + for (HiveStorageInfo hiveStorageInfo : top10Table) { + String qualifiedName=hiveStorageInfo.getTableName(); + String hiveTblGuid = assetService.getHiveTblGuid(qualifiedName); + hiveStorageInfo.setGuid(hiveTblGuid); + } + return Message.ok().data("result",top10Table); + } + + /** + * 搜索hive表 + */ + @RequestMapping(method = RequestMethod.GET, path ="/search") + public Message searchHiveTbl(@RequestParam(required = false) String classification, + @RequestParam(defaultValue = "") String query, + @RequestParam(defaultValue = "") String keyword, + @RequestParam(defaultValue = DEFAULT_LIMIT) int limit, + @RequestParam(defaultValue = DEFAULT_OFFSET) int offset) throws Exception { + List hiveTblSimpleInfoList = assetService.searchHiveTable(classification,query.trim(),limit,offset); + if(hiveTblSimpleInfoList ==null || keyword ==null || keyword.trim().equals("")) { + return Message.ok().data("result",hiveTblSimpleInfoList); + } + else { + Pattern regex = Pattern.compile(keyword); + return Message.ok().data("result",hiveTblSimpleInfoList.stream().filter(ele -> regex.matcher(ele.getOwner()).find()).collect(Collectors.toList())); + } + } + + /** + * 获取单个表的详细信息,包括:基本信息、字段信息 + */ + @RequestMapping(method = RequestMethod.GET, path ="/{guid}/basic") + public Message getHiveTblBasic(@PathVariable String guid) throws Exception { + return Message.ok().data("result",assetService.getHiveTblDetail(guid)); + } + + /** + * 获取表分区信息 + */ + @RequestMapping(method = RequestMethod.GET, path ="/{guid}/partition") + public Message getHiveTblPartition(@PathVariable String guid) throws Exception { + List hiveTblPartition = assetService.getHiveTblPartition(guid); + if (hiveTblPartition.size()>0){ + return Message.ok().data("result",hiveTblPartition); + } + else { + return Message.ok().data("result",null); + } + } + + /** + * 获取表的血缘信息 + */ + @RequestMapping(method = RequestMethod.GET, path ="/{guid}/lineage") + public Message getHiveTblLineage(@PathVariable String guid, + @RequestParam(defaultValue = DEFAULT_DIRECTION) AtlasLineageInfo.LineageDirection direction, + @RequestParam(defaultValue = DEFAULT_DEPTH) int depth) throws Exception { + return Message.ok().data("result",assetService.getHiveTblLineage(guid,direction,depth)); + } + + /** + * 获取表的select语句 + */ + @RequestMapping(method = RequestMethod.GET, path ="/{guid}/select") + public Message getHiveTblSelect(@PathVariable String guid) throws Exception { + return Message.ok().data("result",assetService.getTbSelect(guid)); + } + + /** + * 获取表的create语句 + */ + @RequestMapping(method = RequestMethod.GET, path ="/{guid}/create") + public Message getHiveTblCreate(@PathVariable String guid) throws Exception { + return Message.ok().data("result",assetService.getTbCreate(guid)); + + } + + /** + * 获取分类 + */ + @RequestMapping(method = RequestMethod.GET, path ="/{guid}/classifications") + public Message getClassifications(@PathVariable String guid) throws Exception { + return Message.ok().data("result",assetService.getClassifications(guid)); + } + + /** + * 添加分类 + */ + @Deprecated + @RequestMapping(method = RequestMethod.POST, path ="/{guid}/classifications") + public Message addClassifications(@PathVariable String guid, @RequestBody List classifications) throws Exception { + assetService.addClassifications(guid, classifications); + return Message.ok().data("result","添加成功"); + } + +// /** +// * 删除已有全部旧分类,并添加新分类 +// * linkis-gateway无法正常转换json为list +// * [{"typeName": "test"},{"typeName": "DWD"}] ---> List classifications +// * ["test","DWD"] ---> List typeNames +// */ +// @RequestMapping(method = RequestMethod.PUT, path ="/{guid}/classifications") +// public Message removeAndAddNewClassifications(@PathVariable String guid, @RequestBody List classifications) throws Exception { +// assetService.removeAndAddClassifications(guid, classifications); +// +// return Message.ok().data("result","更新成功"); +// } + + /** + * 删除已有全部旧分类,并添加新分类 + * 支持 {"newClassifications":["test","DWD"]} 非顶层的List数组转换 + */ + @RequestMapping(method = RequestMethod.PUT, path ="/{guid}/classifications") + public Message removeAndAddClassifications(@PathVariable String guid, @RequestBody HiveTblClassificationInfo hiveTblClassificationInfo) throws Exception { + List newClassifications = new ArrayList<>(); + Optional.ofNullable(hiveTblClassificationInfo.getNewClassifications()).orElseGet(()-> { + logger.warn("hive table uid is %s, newClassifications is null",guid); + return new ArrayList<>(); + }).stream().filter(Objects::nonNull).forEach(typeName -> { + AtlasClassification atlasClassification =new AtlasClassification(typeName); + atlasClassification.setPropagate(false); + atlasClassification.setRemovePropagationsOnEntityDelete(true); + newClassifications.add(atlasClassification); + }); + assetService.removeAndAddClassifications(guid, newClassifications); + + return Message.ok().data("result","更新成功"); + } + + /** + * 删除分类 + * @DELETE linkis-gateway 不支持DELETE方式 + */ + @RequestMapping(method = RequestMethod.POST, path ="/{guid}/classification/{classificationName}") + public Message deleteClassification(@PathVariable String guid, @PathVariable final String classificationName) throws Exception { + assetService.deleteClassification(guid, classificationName); + + return Message.ok().data("result","删除成功"); + } +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/AssetService.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/AssetService.java new file mode 100644 index 000000000..d42190e27 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/AssetService.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.data.asset.service; + +import com.webank.wedatasphere.dss.data.asset.entity.*; +import com.webank.wedatasphere.dss.data.common.exception.DataGovernanceException; +import com.webank.wedatasphere.dss.data.common.atlas.AtlasClassificationV2.AtlasClassificationsV2; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.lineage.AtlasLineageInfo; + +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface AssetService { + public Map getHiveSummary() throws DataGovernanceException; + + public List searchHiveTable(String classification, String query, + int limit, int offset) throws DataGovernanceException; + + public HiveTblDetailInfo getHiveTblDetail(String guid) throws DataGovernanceException; + + public List getHiveTblPartition(String guid) throws DataGovernanceException; + + public String getHiveTblGuid(String qualifiedName) throws DataGovernanceException; + + public String getTbSelect(String guid) throws DataGovernanceException; + + public String getTbCreate(String guid) throws DataGovernanceException; + + public void modifyComment(String guid, String commentStr) throws DataGovernanceException; + + public void bulkModifyComment(Map commentMap) throws DataGovernanceException; + + public void setLabels(String guid, Set labels) throws DataGovernanceException; + + public void removeLabels(String guid, Set labels) throws DataGovernanceException; + + /** + * 获取表实体的血缘信息 + */ + public AtlasLineageInfo getHiveTblLineage(final String guid, final AtlasLineageInfo.LineageDirection direction, final int depth) throws DataGovernanceException; + + public List getTop10Table() throws DataGovernanceException, SQLException; + + public void addClassifications(String guid, List classifications) throws DataGovernanceException; + + public void deleteClassification(String guid, String classificationName) throws DataGovernanceException; + + public void deleteClassifications(String guid, List classifications) throws DataGovernanceException; + + public void updateClassifications(String guid, List classifications) throws DataGovernanceException; + + /** + * 为实体删除已有的分类,添加新的分类 + */ + public void removeAndAddClassifications(String guid, List newClassifications) throws DataGovernanceException; + + public AtlasClassificationsV2 getClassifications(String guid) throws DataGovernanceException; +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/WorkspaceInfoService.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/WorkspaceInfoService.java new file mode 100644 index 000000000..b21828026 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/WorkspaceInfoService.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.data.asset.service; + + +import com.webank.wedatasphere.dss.data.common.exception.DataGovernanceException; + + +import java.util.List; + +public interface WorkspaceInfoService { + + public List getWorkspaceUsers(int workspaceId,String search) throws DataGovernanceException; + +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/impl/AssetServiceImpl.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/impl/AssetServiceImpl.java new file mode 100644 index 000000000..31bf580ff --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/impl/AssetServiceImpl.java @@ -0,0 +1,518 @@ +package com.webank.wedatasphere.dss.data.asset.service.impl; + +import com.google.gson.internal.LinkedTreeMap; +import com.webank.wedatasphere.dss.data.asset.dao.MetaInfoMapper; +import com.webank.wedatasphere.dss.data.asset.dao.impl.MetaInfoMapperImpl; +import com.webank.wedatasphere.dss.data.asset.entity.HivePartInfo; +import com.webank.wedatasphere.dss.data.asset.entity.HiveStorageInfo; +import com.webank.wedatasphere.dss.data.asset.entity.HiveTblDetailInfo; +import com.webank.wedatasphere.dss.data.asset.entity.HiveTblSimpleInfo; +import com.webank.wedatasphere.dss.data.asset.service.AssetService; +import com.webank.wedatasphere.dss.data.common.atlas.AtlasClassificationV2; +import com.webank.wedatasphere.dss.data.common.atlas.AtlasService; +import com.webank.wedatasphere.dss.data.common.conf.AtlasConf; +import com.webank.wedatasphere.dss.data.common.exception.DAOException; +import com.webank.wedatasphere.dss.data.common.exception.DataGovernanceException; +import com.webank.wedatasphere.dss.data.common.utils.DateUtil; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@Service +public class AssetServiceImpl implements AssetService { + private static final Logger logger = LoggerFactory.getLogger(AssetServiceImpl.class); + + private AtlasService atlasService; + private MetaInfoMapper metaInfoMapper; + + public AssetServiceImpl(AtlasService atlasService) { + this.atlasService = atlasService; + this.metaInfoMapper = new MetaInfoMapperImpl(); + } + + @Override + public Map getHiveSummary() throws DataGovernanceException { + try { + Map result = new HashMap<>(); + + result.put("hiveDb", atlasService.getHiveDbCnt()); + result.put("hiveTable", atlasService.getHiveTableCnt()); + result.put("hiveStore", metaInfoMapper.getTableStorage()); + + return result; + } catch (AtlasServiceException | DAOException | SQLException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public List searchHiveTable(String classification, String query, + int limit, int offset) throws DataGovernanceException { + List atlasEntityHeaders = null; + try { + atlasEntityHeaders = atlasService.searchHiveTable(classification, "*" + query + "*", true, limit, offset); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + + if (atlasEntityHeaders != null) { + //columns 根据keyword来正则匹配过滤 + Pattern regex = Pattern.compile(query); + return atlasEntityHeaders.parallelStream().filter(Objects::nonNull).map(atlasEntityHeader -> { + HiveTblSimpleInfo hiveTblSimpleInfo = new HiveTblSimpleInfo(); + hiveTblSimpleInfo.setGuid(atlasEntityHeader.getGuid()); + hiveTblSimpleInfo.setName(stringValueOfObject(atlasEntityHeader.getAttribute("name"))); + String qualifiedName =stringValueOfObject(atlasEntityHeader.getAttribute("qualifiedName")); + hiveTblSimpleInfo.setQualifiedName(qualifiedName); + hiveTblSimpleInfo.setOwner(stringValueOfObject(atlasEntityHeader.getAttribute("owner"))); + Object createTime = atlasEntityHeader.getAttribute("createTime"); + if (createTime != null) { + hiveTblSimpleInfo.setCreateTime(DateUtil.unixToTimeStr((Double) createTime)); + } + if(null != qualifiedName && qualifiedName.split("\\.").length >0){ + String dbName = qualifiedName.split("\\.")[0]; + hiveTblSimpleInfo.setDbName(dbName); + } + hiveTblSimpleInfo.setLabels(atlasEntityHeader.getLabels()); + + try { + AtlasEntity atlasEntity = atlasService.getHiveTblByGuid(atlasEntityHeader.getGuid()); + + //comment + hiveTblSimpleInfo.setComment(stringValueOfObject(atlasEntity.getAttribute("comment"))); + List> atlasRelatedObjectIdListForColumns = (List>)atlasEntity.getRelationshipAttribute("columns"); + if(null != query && !query.trim().equalsIgnoreCase("")) { + hiveTblSimpleInfo.setColumns(atlasRelatedObjectIdListForColumns.stream().map(columnMap -> columnMap.getOrDefault("displayText","").toString()) + .filter(columnName -> regex.matcher(columnName).find()).collect(Collectors.toList())); + } + //classifications + List classificationInfoList = getClassificationInfoList(atlasEntity); + hiveTblSimpleInfo.setClassifications(classificationInfoList); + } catch (AtlasServiceException ex) { + logger.error(ex.getMessage()); + } + + return hiveTblSimpleInfo; + }).collect(Collectors.toList()); + } + return null; + } + + private String stringValueOfObject(Object obj){ + if(null !=obj) { + return obj.toString(); + } + else { + return null; + } + } + + @Override + public String getHiveTblGuid(String qualifiedName) throws DataGovernanceException{ + qualifiedName = qualifiedName + "@" + AtlasConf.ATLAS_CLUSTER_NAME.getValue(); + Map uniqAttributes =new HashMap<>(); + uniqAttributes.put("qualifiedName",qualifiedName); + try{ + AtlasEntity atlasEntity = atlasService.getHiveTblByAttribute(uniqAttributes,true,true); + if(atlasEntity == null){ + logger.warn(String.format("%s not exist in atlas", qualifiedName)); + return null; + } + else { + return atlasEntity.getGuid(); + } + } catch (AtlasServiceException ex) { + logger.warn(String.format("%s not exist in atlas", qualifiedName)); + return null; + } + } + + @Override + public HiveTblDetailInfo getHiveTblDetail(String guid) throws DataGovernanceException { + try { + AtlasEntity atlasEntity = atlasService.getHiveTblByGuid(guid); + + HiveTblDetailInfo hiveTblDetailInfo = new HiveTblDetailInfo(); + hiveTblDetailInfo.setBasic(getBasicInfo(guid, atlasEntity)); + hiveTblDetailInfo.setColumns(getBasicColumnInfoList(atlasEntity)); + hiveTblDetailInfo.setPartitionKeys(getPartitionColumnInfoList(atlasEntity)); + hiveTblDetailInfo.setClassifications(getClassificationInfoList(atlasEntity)); + + return hiveTblDetailInfo; + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + } + + private HiveTblDetailInfo.HiveTblBasicInfo getBasicInfo(String guid, AtlasEntity atlasEntity) throws AtlasServiceException { + Map hiveTblAttributesMap = atlasService.getHiveTblAttributesByGuid(guid); + Boolean isPartTable = (Boolean) hiveTblAttributesMap.get("isPartition"); + int storage = 0; + String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; + String tableName = db_name.split("\\.")[1]; + String dbName = db_name.split("\\.")[0]; + try { + storage = metaInfoMapper.getTableInfo(dbName, tableName, isPartTable); + } catch (SQLException e) { + e.printStackTrace(); + } + + HiveTblDetailInfo.HiveTblBasicInfo basic = new HiveTblDetailInfo.HiveTblBasicInfo(); + basic.setName(tableName); + basic.setOwner(String.valueOf(atlasEntity.getAttributes().getOrDefault("owner","NULL"))); + basic.setCreateTime(new java.text.SimpleDateFormat("yyyy MM-dd HH:mm:ss").format(atlasEntity.getCreateTime())); + basic.setStore(String.valueOf(storage)); + basic.setComment(String.valueOf(atlasEntity.getAttributes().getOrDefault("comment","NULL"))); + Set labels = atlasEntity.getLabels(); + basic.setLabels(labels); + basic.setIsParTbl(isPartTable); + basic.setGuid(guid); + basic.setTableType(hiveTblAttributesMap.getOrDefault("tableType","NULL").toString()); + basic.setLocation(hiveTblAttributesMap.getOrDefault("location","NULL").toString()); + + return basic; + } + + private List getBasicColumnInfoList(AtlasEntity atlasEntity) throws AtlasServiceException { + List guids = new ArrayList<>(); + List> columns = (List>) atlasEntity.getAttributes().get("columns"); + for (LinkedTreeMap column : columns) { + guids.add(column.get("guid")); + } + + List hiveColumnInfos = new ArrayList<>(); + if (guids.size() > 0) { + List hiveColumnsByGuids = atlasService.getHiveColumnsByGuids(guids); + for (AtlasEntity hiveColumnsByGuid : hiveColumnsByGuids) { + HiveTblDetailInfo.HiveColumnInfo hiveColumnInfo = new HiveTblDetailInfo.HiveColumnInfo(); + hiveColumnInfo.setName(String.valueOf(hiveColumnsByGuid.getAttributes().get("name"))); + hiveColumnInfo.setType(String.valueOf(hiveColumnsByGuid.getAttributes().get("type"))); + hiveColumnInfo.setComment(String.valueOf(hiveColumnsByGuid.getAttributes().get("comment"))); + hiveColumnInfo.setGuid(hiveColumnsByGuid.getGuid()); + hiveColumnInfos.add(hiveColumnInfo); + } + } + return hiveColumnInfos; + } + + private List getPartitionColumnInfoList(AtlasEntity atlasEntity) throws AtlasServiceException { + List partguids = new ArrayList<>(); + List> partitionKeys = (List>) atlasEntity.getAttributes().get("partitionKeys"); + for (LinkedTreeMap column : partitionKeys) { + partguids.add(column.get("guid")); + } + List partitionColumns = new ArrayList<>(); + if (partguids.size() > 0) { + List hivePartColumnsByGuids = atlasService.getHiveColumnsByGuids(partguids); + for (AtlasEntity hiveColumnsByGuid : hivePartColumnsByGuids) { + HiveTblDetailInfo.HiveColumnInfo hiveColumnInfo = new HiveTblDetailInfo.HiveColumnInfo(); + hiveColumnInfo.setName(String.valueOf(hiveColumnsByGuid.getAttributes().get("name"))); + hiveColumnInfo.setType(String.valueOf(hiveColumnsByGuid.getAttributes().get("type"))); + hiveColumnInfo.setComment(String.valueOf(hiveColumnsByGuid.getAttributes().get("comment"))); + hiveColumnInfo.setGuid(hiveColumnsByGuid.getGuid()); + partitionColumns.add(hiveColumnInfo); + } + } + return partitionColumns; + } + + private List getClassificationInfoList(AtlasEntity atlasEntity) throws AtlasServiceException { + if(atlasEntity.getClassifications() ==null) { + return null; + } + else { + List hiveClassificationInfoList =new ArrayList<>(); + String typeName =null; + AtlasClassificationDef atlasClassificationDef =null; + + List atlasClassificationList = atlasEntity.getClassifications(); + for (AtlasClassification atlasClassification : atlasClassificationList) { + typeName = atlasClassification.getTypeName(); + atlasClassificationDef = getClassificationDefByName(typeName); + hiveClassificationInfoList.add( + new HiveTblDetailInfo.HiveClassificationInfo(typeName,atlasClassificationDef.getSuperTypes(),atlasClassificationDef.getSubTypes())); + } + return hiveClassificationInfoList; + } + } + + private AtlasClassificationDef getClassificationDefByName(String name) throws DataGovernanceException { + try { + return atlasService.getClassificationDefByName(name); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public List getHiveTblPartition(String guid) throws DataGovernanceException { + try { + AtlasEntity atlasEntity = atlasService.getHiveTblByGuid(guid); + String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; + String tableName = db_name.split("\\.")[1]; + String dbName = db_name.split("\\.")[0]; + List hivePartInfo = new ArrayList<>(); + try { + hivePartInfo = metaInfoMapper.getPartInfo(dbName, tableName); + } catch (SQLException e) { + e.printStackTrace(); + } + return hivePartInfo; + + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + } + + @Override + public String getTbSelect(String guid) throws DataGovernanceException { + try { + AtlasEntity atlasEntity = atlasService.getHiveTblByGuid(guid); + String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; + String tableName = db_name.split("\\.")[1]; + List guids = new ArrayList<>(); + List> columns = (List>) atlasEntity.getAttributes().get("columns"); + for (LinkedTreeMap column : columns) { + guids.add(column.get("guid")); + } + List fields = new ArrayList<>(); + List hiveColumnsByGuids = atlasService.getHiveColumnsByGuids(guids); + for (AtlasEntity hiveColumnsByGuid : hiveColumnsByGuids) { + fields.add((String) hiveColumnsByGuid.getAttributes().get("name")); + } + Map hiveTblAttributesMap = atlasService.getHiveTblAttributesByGuid(guid); + Boolean isPartTable = (Boolean) hiveTblAttributesMap.get("isPartition"); + if (isPartTable == true) { + List partguids = new ArrayList<>(); + List> partitionKeys = (List>) atlasEntity.getAttributes().get("partitionKeys"); + for (LinkedTreeMap column : partitionKeys) { + partguids.add(column.get("guid")); + } + List hiveColumnsByGuids1 = atlasService.getHiveColumnsByGuids(partguids); + for (AtlasEntity entity : hiveColumnsByGuids1) { + fields.add((String) entity.getAttributes().get("name")); + } + } + StringBuilder sql = new StringBuilder(); + sql.append("SELECT @$ "); + for (int i = 0; i < fields.size() - 1; i++) { + sql.append(fields.get(i)).append(", @$ "); + } + sql.append(fields.get(fields.size() - 1)).append(" @$ from ").append(tableName); + return sql.toString(); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + } + + @Override + public String getTbCreate(String guid) throws DataGovernanceException { + try { + StringBuilder sql = new StringBuilder(); + AtlasEntity atlasEntity = atlasService.getHiveTblByGuid(guid); + String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; + String tableName = db_name.split("\\.")[1]; + List guids = new ArrayList<>(); + List> columns = (List>) atlasEntity.getAttributes().get("columns"); + for (LinkedTreeMap column : columns) { + guids.add(column.get("guid")); + } + sql.append("CREATE TABLE IF NOT EXISTS ").append(tableName).append(" @$ ( @$ "); + List fields = new ArrayList<>(); + List hiveColumnsByGuids = atlasService.getHiveColumnsByGuids(guids); + for (int i = 0; i < hiveColumnsByGuids.size(); i++) { + if (i < hiveColumnsByGuids.size() - 1) { + AtlasEntity hiveColumnsByGuid = hiveColumnsByGuids.get(i); + StringBuilder sb = new StringBuilder(); + sb.append((String) hiveColumnsByGuid.getAttributes().get("name")).append(" ").append((String) hiveColumnsByGuid.getAttributes().get("type")); + if (hiveColumnsByGuid.getAttributes().get("comment") == null) { + sb.append(", "); + } else { + sb.append(" COMMENT '").append(hiveColumnsByGuid.getAttributes().get("comment")).append("',"); + } + fields.add(sb.append(" @$ ").toString()); + } else { + AtlasEntity hiveColumnsByGuid = hiveColumnsByGuids.get(i); + StringBuilder sb = new StringBuilder(); + sb.append((String) hiveColumnsByGuid.getAttributes().get("name")).append(" ").append((String) hiveColumnsByGuid.getAttributes().get("type")); + if (hiveColumnsByGuid.getAttributes().get("comment") == null) { + sb.append(" "); + } else { + sb.append(" COMMENT '").append(hiveColumnsByGuid.getAttributes().get("comment")).append("' "); + } + fields.add(sb.append(" @$ ").toString()); + } + } + for (String field : fields) { + sql.append(field); + } + sql.append(") @$ "); + Map hiveTblAttributesMap = atlasService.getHiveTblAttributesByGuid(guid); + Boolean isPartTable = (Boolean) hiveTblAttributesMap.get("isPartition"); + if (isPartTable == true) { + sql.append("PARTITIONED BY @$ ( @$ "); + List partguids = new ArrayList<>(); + List> partitionKeys = (List>) atlasEntity.getAttributes().get("partitionKeys"); + for (LinkedTreeMap column : partitionKeys) { + partguids.add(column.get("guid")); + } + List keyFields = new ArrayList<>(); + List hiveColumnsByGuids1 = atlasService.getHiveColumnsByGuids(partguids); + for (int i = 0; i < hiveColumnsByGuids1.size(); i++) { + if (i < hiveColumnsByGuids1.size() - 1) { + AtlasEntity entity = hiveColumnsByGuids1.get(i); + StringBuilder sb = new StringBuilder(); + sb.append((String) entity.getAttributes().get("name")).append(" ").append((String) entity.getAttributes().get("type")); + if (entity.getAttributes().get("comment") == null) { + sb.append(", "); + } else { + sb.append(" COMMENT '").append(entity.getAttributes().get("comment")).append("', "); + } + keyFields.add(sb.append(" @$ ").toString()); + } else { + AtlasEntity entity = hiveColumnsByGuids1.get(i); + StringBuilder sb = new StringBuilder(); + sb.append((String) entity.getAttributes().get("name")).append(" ").append((String) entity.getAttributes().get("type")); + if (entity.getAttributes().get("comment") == null) { + sb.append(" "); + } else { + sb.append(" COMMENT '").append(entity.getAttributes().get("comment")).append("' "); + } + keyFields.add(sb.append(" @$ ").toString()); + } + } + for (int i = 0; i < keyFields.size(); i++) { + sql.append(keyFields.get(i)); + } + sql.append(")"); + } + return sql.toString(); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + } + + @Override + public void modifyComment(String guid, String commentStr) throws DataGovernanceException { + try { + atlasService.modifyComment(guid, commentStr); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + } + + @Override + public void bulkModifyComment(Map commentMap) throws DataGovernanceException { + commentMap.keySet().forEach(key -> { + try { + atlasService.modifyComment(key, commentMap.get(key)); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + }); + + } + + @Override + public void setLabels(String guid, Set labels) throws DataGovernanceException { + try { + atlasService.setLabels(guid, labels); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + } + + @Override + public void removeLabels(String guid, Set labels) throws DataGovernanceException { + try { + atlasService.removeLabels(guid, labels); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(ex.getMessage()); + } + } + + @Override + public AtlasLineageInfo getHiveTblLineage(final String guid, final AtlasLineageInfo.LineageDirection direction, final int depth) throws DataGovernanceException { + try { + return atlasService.getLineageInfo(guid, direction, depth); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public List getTop10Table() throws DataGovernanceException, SQLException { + return metaInfoMapper.getTop10Table(); + } + + @Override + public void addClassifications(String guid, List classifications) throws DataGovernanceException { + try { + atlasService.addClassifications(guid,classifications); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public void deleteClassification(String guid, String classificationName) throws DataGovernanceException{ + try { + atlasService.deleteClassification(guid, classificationName); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public void deleteClassifications(String guid, List classifications) throws DataGovernanceException { + try { + atlasService.deleteClassifications(guid, classifications); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public void updateClassifications(String guid, List classifications) throws DataGovernanceException { + try { + atlasService.updateClassifications(guid, classifications); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public void removeAndAddClassifications(String guid, List newClassifications) throws DataGovernanceException { + try { + atlasService.removeAndAddClassifications(guid, newClassifications); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public AtlasClassificationV2.AtlasClassificationsV2 getClassifications(String guid) throws DataGovernanceException { + try { + return atlasService.getClassifications(guid); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } +} diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/impl/WorkspaceInfoServiceImpl.java b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/impl/WorkspaceInfoServiceImpl.java new file mode 100644 index 000000000..243ffe900 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/impl/WorkspaceInfoServiceImpl.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.data.asset.service.impl; + +import com.webank.wedatasphere.dss.data.asset.dao.WorkspaceInfoMapper; +import com.webank.wedatasphere.dss.data.common.exception.DataGovernanceException; +import com.webank.wedatasphere.dss.data.asset.service.WorkspaceInfoService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.*; + +@Service +public class WorkspaceInfoServiceImpl implements WorkspaceInfoService { + + @Autowired + private WorkspaceInfoMapper workspaceInfoMapper; + + + @Override + public List getWorkspaceUsers(int workspaceId,String search) throws DataGovernanceException { + + List workspaceUsers = workspaceInfoMapper.getWorkspaceUsersName(workspaceId,search); + return workspaceUsers; + } +} diff --git a/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml b/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml new file mode 100644 index 000000000..154332245 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml @@ -0,0 +1,191 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-data-classification-server + + + 2.1.0 + + + + com.webank.wedatasphere.dss + dss-data-governance-common + ${dss.version} + compile + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + org.springframework.cloud + spring-cloud-netflix + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + javax.ws.rs + javax.ws.rs-api + + + + + org.apache.commons + commons-math3 + provided + + + xstream + com.thoughtworks.xstream + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + ${spring.cloud.version} + provided + + + logback-classic + ch.qos.logback + + + log4j-to-slf4j + org.apache.logging.log4j + + + gson + com.google.code.gson + + + jsr311-api + javax.ws.rs + + + xstream + com.thoughtworks.xstream + + + commons-math + org.apache.commons + + + jackson-core + com.fasterxml.jackson.core + + + spring-boot-autoconfigure + org.springframework.boot + + + spring-boot-starter-aop + org.springframework.boot + + + spring-boot-starter + org.springframework.boot + + + spring-boot-starter-cache + org.springframework.boot + + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-classification-server/src/main/assembly/distribution.xml b/dss-apps/dss-data-governance/dss-data-classification-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..46d83d32f --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-classification-server/src/main/assembly/distribution.xml @@ -0,0 +1,46 @@ + + + + dss-data-asset-server + + dir + + true + dss-data-asset-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/restful/ClassificationRestful.java b/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/restful/ClassificationRestful.java new file mode 100644 index 000000000..926995ea2 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/restful/ClassificationRestful.java @@ -0,0 +1,98 @@ +package com.webank.wedatasphere.dss.data.classification.restful; + +import com.webank.wedatasphere.dss.data.classification.service.ClassificationService; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@RestController +@RequestMapping(path = "/dss/data/governance/classification", produces = {"application/json"}) +@Slf4j +@AllArgsConstructor +public class ClassificationRestful { + private static final Logger logger = LoggerFactory.getLogger(ClassificationRestful.class); + + private ClassificationService classificationService; + + /** + * 获取所有分类 + */ + @RequestMapping(method = RequestMethod.GET, path ="/all") + public Message getClassificationDef() throws Exception { + return Message.ok().data("result", classificationService.getClassificationDef()); + } + + /** + * 根据名称获取分类,不包括包括子分类 + */ + @RequestMapping(method = RequestMethod.GET, path ="/{name}") + public Message getClassificationDef(@PathVariable String name) throws Exception { + return Message.ok().data("result",classificationService.getClassificationDefByName(name)); + } + + /** + * 根据名称获取分类,包括一级子分类 + * keyword 按照分类名称模糊搜索 + */ + @RequestMapping(method = RequestMethod.GET, path ="/{name}/subtypes") + public Message getClassificationDefList(@PathVariable String name, + @RequestParam(defaultValue = "") String keyword) throws Exception { + List atlasClassificationDefList = classificationService.getClassificationDefListByName(name); + if(atlasClassificationDefList ==null || keyword ==null || keyword.trim().equalsIgnoreCase("")) { + return Message.ok().data("result", atlasClassificationDefList); + } + else{ + Pattern regex = Pattern.compile(keyword); + return Message.ok().data("result",atlasClassificationDefList.stream().filter(ele -> regex.matcher(ele.getName()).find()).collect(Collectors.toList())); + } + } + + /** + * 获取所有分层的一级子类型,包括系统预置分层和用户自定义分层 + */ + @RequestMapping(method = RequestMethod.GET, path ="/layers/all") + public Message getClassificationDefListForLayer() throws Exception { + return Message.ok().data("result",classificationService.getClassificationDefListForLayer()); + } + + /** + * 创建新的分类 + */ + @RequestMapping(method = RequestMethod.POST, path ="/batch") + public Message createClassificationDef(@RequestBody AtlasTypesDef typesDef) throws Exception { + return Message.ok().data("result", classificationService.createAtlasTypeDefs(typesDef)); + } + + /** + * 更新的分类 + */ + @RequestMapping(method = RequestMethod.PUT, path ="/batch") + public Message updateClassificationDef(@RequestBody AtlasTypesDef typesDef) throws Exception { + classificationService.updateAtlasTypeDefs(typesDef); + return Message.ok().data("result","更新成功" ); + } + + /** + * 删除分类 + * @DELETE linkis-gateway 不支持DELETE方式 + */ + @RequestMapping(method = RequestMethod.POST, path ="/{name}/delete") + public Message deleteClassificationDef(@PathVariable String name) throws Exception { + classificationService.deleteClassificationDefByName(name); + return Message.ok().data("result","删除成功"); + } +} diff --git a/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/service/ClassificationService.java b/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/service/ClassificationService.java new file mode 100644 index 000000000..1fabe8b2d --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/service/ClassificationService.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.dss.data.classification.service; + +import com.webank.wedatasphere.dss.data.common.exception.DataGovernanceException; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; + +import java.util.List; + +public interface ClassificationService { + public AtlasTypesDef getClassificationDef( ) throws DataGovernanceException; + + public AtlasClassificationDef getClassificationDefByName(String name) throws DataGovernanceException; + + public List getClassificationDefListByName(String name) throws DataGovernanceException; + + public List getClassificationDefListForLayer() throws DataGovernanceException; + + public AtlasTypesDef createAtlasTypeDefs(final AtlasTypesDef typesDef) throws DataGovernanceException; + + public void updateAtlasTypeDefs(final AtlasTypesDef typesDef) throws DataGovernanceException; + + public void deleteClassificationDefByName(String name) throws DataGovernanceException; + + +} diff --git a/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/service/impl/ClassificationServiceImpl.java b/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/service/impl/ClassificationServiceImpl.java new file mode 100644 index 000000000..24753e769 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-classification-server/src/main/java/com/webank/wedatasphere/dss/data/classification/service/impl/ClassificationServiceImpl.java @@ -0,0 +1,134 @@ +package com.webank.wedatasphere.dss.data.classification.service.impl; + +import com.sun.jersey.api.client.ClientResponse; +import com.webank.wedatasphere.dss.data.classification.service.ClassificationService; +import com.webank.wedatasphere.dss.data.common.atlas.AtlasService; +import com.webank.wedatasphere.dss.data.common.conf.AtlasConf; +import com.webank.wedatasphere.dss.data.common.exception.DataGovernanceException; +import lombok.AllArgsConstructor; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Service +@AllArgsConstructor +public class ClassificationServiceImpl implements ClassificationService { + private static final Logger logger = LoggerFactory.getLogger(ClassificationServiceImpl.class); + private static final String ATLAS_ERROR_CODE ="errorCode"; + + private AtlasService atlasService; + + /** + * 判断系统预定义最顶层分类是否已存在,如果不存在,则需要进行初始化 + */ + @PostConstruct + private void initializeSystemClassificationList(){ + logger.info("@@@@@@@@@@@@@@ ==> initializeSystemClassificationList"); + + initializeClassification(AtlasConf.ATLAS_CLASSIFICATION_SUBJECT.getValue(),"数仓主题域"); + initializeClassification(AtlasConf.ATLAS_CLASSIFICATION_LAYER_SYSTEM.getValue(),"系统预定义分层"); + initializeClassification(AtlasConf.ATLAS_CLASSIFICATION_LAYER_USER.getValue(),"用户自定义分层"); + + logger.info("@@@@@@@@@@@@@@ <== initializeSystemClassificationList"); + } + + private void initializeClassification(String name, String description){ + AtlasClassificationDef atlasClassificationDef =null; + AtlasTypesDef atlasTypesDef =null; + + try { + atlasClassificationDef = atlasService.getClassificationDefByName(name); + if(atlasClassificationDef !=null) { + logger.info(String.format("@@@@@@@@@@@@@@ === this classification [name=%s] exists, ignore it", name)); + } + }catch (AtlasServiceException ex) { + logger.info("@@@@@@@@@@ error:" + ex.getMessage()); + // 404 + if (ClientResponse.Status.NOT_FOUND ==ex.getStatus()) { + logger.info(String.format("@@@@@@@@@@@@@@ === this classification [name=%s] don't exist, create it...", name)); + logger.info(String.format("@@@@@@@@@@@@@@ ==> initializeClassification, name=%s", name)); + atlasTypesDef = new AtlasTypesDef(); + atlasClassificationDef = new AtlasClassificationDef(name, description); + atlasTypesDef.getClassificationDefs().add(atlasClassificationDef); + this.createAtlasTypeDefs(atlasTypesDef); + logger.info(String.format("@@@@@@@@@@@@@@ <== initializeClassification, name=%s", name)); + } + else { + throw new DataGovernanceException(ex.getMessage()); + } + } + } + + + @Override + public AtlasTypesDef getClassificationDef() throws DataGovernanceException { + try { + return atlasService.getClassificationDef(); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public AtlasClassificationDef getClassificationDefByName(String name) throws DataGovernanceException { + try { + return atlasService.getClassificationDefByName(name); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public List getClassificationDefListByName(String name) throws DataGovernanceException { + try { + return atlasService.getClassificationDefListByName(name); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public List getClassificationDefListForLayer() throws DataGovernanceException { + try { + return atlasService.getClassificationDefListForLayer(); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public AtlasTypesDef createAtlasTypeDefs(final AtlasTypesDef typesDef) throws DataGovernanceException { + try { + return atlasService.createAtlasTypeDefs(typesDef); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public void updateAtlasTypeDefs(AtlasTypesDef typesDef) throws DataGovernanceException { + try { + atlasService.updateAtlasTypeDefs(typesDef); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(exception.getMessage()); + } + } + + @Override + public void deleteClassificationDefByName(String name) throws DataGovernanceException { + try { + atlasService.deleteTypedefByName(name); + } catch (AtlasServiceException exception) { + if(ClientResponse.Status.CONFLICT ==exception.getStatus()){ + throw new DataGovernanceException("该分类(主题域/分层)已经与表关联,无法删除"); + } + throw new DataGovernanceException(exception.getMessage()); + } + } +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml b/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml new file mode 100644 index 000000000..bdd9f0c2f --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml @@ -0,0 +1,139 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-data-governance-common + + + 2.1.0 + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + org.springframework.cloud + spring-cloud-netflix + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + + + org.apache.atlas + atlas-client-v2 + ${atlas.version} + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + + + org.projectlombok + lombok + 1.18.16 + compile + + + com.alibaba + druid + 1.1.9 + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/assembly/distribution.xml b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/assembly/distribution.xml new file mode 100644 index 000000000..46d83d32f --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/assembly/distribution.xml @@ -0,0 +1,46 @@ + + + + dss-data-asset-server + + dir + + true + dss-data-asset-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasClassificationV2.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasClassificationV2.java new file mode 100644 index 000000000..561fb757d --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasClassificationV2.java @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.dss.data.common.atlas; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.model.PList; +import org.apache.atlas.model.SearchFilter; +import org.apache.atlas.model.instance.AtlasClassification; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlSeeAlso; +import java.util.List; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +public class AtlasClassificationV2 extends AtlasClassification { + // org.codehaus.jackson.map.JsonMappingException: Conflicting getter definitions for property "propagate" + // https://stackoverflow.com/questions/20624891/stuck-on-org-codehaus-jackson-map-jsonmappingexception-conflicting-getter-defi + @Override + @JsonIgnore + public Boolean isPropagate() { + return super.getPropagate(); + } + + /** + * REST serialization friendly list. + */ + @JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.PROPERTY) + @XmlSeeAlso(AtlasClassification.class) + public static class AtlasClassificationsV2 extends PList { + private static final long serialVersionUID = 1L; + + public AtlasClassificationsV2() { + super(); + } + + public AtlasClassificationsV2(List list) { + super(list); + } + + public AtlasClassificationsV2(List list, long startIndex, int pageSize, long totalCount, + SearchFilter.SortType sortType, String sortBy) { + super(list, startIndex, pageSize, totalCount, sortType, sortBy); + } + } +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasClient.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasClient.java new file mode 100644 index 000000000..ce7c57dd2 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasClient.java @@ -0,0 +1,274 @@ +package com.webank.wedatasphere.dss.data.common.atlas; + +import com.sun.jersey.core.util.MultivaluedMapImpl; +import org.apache.atlas.AtlasClientV2; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.SortOrder; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.MapUtils; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class AtlasClient extends AtlasClientV2 { + private static final String PREFIX_ATTR = "attr:"; + + private static final String ENTITIES_API = BASE_URI + "entities/"; + private static final String UPDATE_ENTITY_ATTR_API = ENTITY_API + "guid/"; + private static final String SET_ENTITY_LABELS_BY_GUID_TEMPLATE = ENTITY_API + "/guid/%s/labels"; + private static final String DELETE_ENTITY_LABELS_BY_GUID_TEMPLATE = ENTITY_API + "/guid/%s/labels"; + private static final String GET_ENTITY_HEADER_BY_GUID_TEMPLATE = ENTITY_API + "/guid/{guid}/header"; + + private static final String TYPEDEF_BY_NAME = TYPES_API + "typedef/name/"; + private static final String TYPEDEFS_API = TYPES_API + "typedefs/"; + private static final String GET_CLASSIFICATION_BY_NAME_TEMPLATE = TYPES_API + "classificationdef/name/%s"; + + public AtlasClient(String[] baseUrl, String[] basicAuthUserNamePassword) { + super(baseUrl, basicAuthUserNamePassword); + } + + /** + * 获取所有的hive db实体 + */ + public String getHiveDbs() throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("type", "hive_db"); + queryParams.add("property", AtlasEntity.KEY_STATUS); + queryParams.add("value", AtlasEntity.Status.ACTIVE.name()); + return callAPI(API_V3.GET_ENTITIES, String.class, queryParams); + } + + /** + * 获取所有的hive table实体 + */ + public String getHiveTables() throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("type", "hive_table"); + queryParams.add("property", AtlasEntity.KEY_STATUS); + queryParams.add("value", AtlasEntity.Status.ACTIVE.name()); + return callAPI(API_V3.GET_ENTITIES, String.class, queryParams); + } + + /** + * 获取表实体的血缘信息 + */ + public String getLineageInfoForString(final String guid, final AtlasLineageInfo.LineageDirection direction, final int depth) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("direction", direction.toString()); + queryParams.add("depth", String.valueOf(depth)); + + return callAPI(API_V2.LINEAGE_INFO, String.class, queryParams, guid); + } + + /** + * 获取实体信息 + */ + public String getEntityByGuidForString(String guid) throws AtlasServiceException { + return getEntityByGuidForString(guid, false, false); + } + + /** + * 获取实体信息 + */ + public String getEntityByGuidForString(String guid, boolean minExtInfo, boolean ignoreRelationships) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + + queryParams.add("minExtInfo", String.valueOf(minExtInfo)); + queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationships)); + + return callAPI(API_V2.GET_ENTITY_BY_GUID, String.class, queryParams, guid); + } + + /** + * 通过type 和 属性查一个entity + */ + public String getEntityByAttributeForString(String type, Map uniqAttributes, boolean minExtInfo, boolean ignoreRelationship) throws AtlasServiceException { + MultivaluedMap queryParams = attributesToQueryParams(uniqAttributes); + + queryParams.add("minExtInfo", String.valueOf(minExtInfo)); + queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationship)); + + return callAPI(API_V2.GET_ENTITY_BY_ATTRIBUTE, String.class, queryParams, type); + } + + + public String getEntitiesByGuidsForString(List guids) throws AtlasServiceException { + return getEntitiesByGuidsForString(guids, false, false); + } + + public String getEntitiesByGuidsForString(List guids, boolean minExtInfo, boolean ignoreRelationships) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + + queryParams.put("guid", guids); + queryParams.add("minExtInfo", String.valueOf(minExtInfo)); + queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationships)); + + return callAPI(API_V2.GET_ENTITIES_BY_GUIDS, String.class, queryParams); + } + + /** + * 获取实体基本信息 + */ + public String getHeaderByIdForString(String guid) throws AtlasServiceException { + API api = new API(String.format(GET_ENTITY_HEADER_BY_GUID_TEMPLATE, guid), HttpMethod.GET, Response.Status.OK); + return callAPI(api, String.class,null); + } + + /** + * 根据关键字检索实体 + */ + public String basicSearchForString(final String typeName, final String classification, final String query, + final boolean excludeDeletedEntities, final int limit, final int offset) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("typeName", typeName); + queryParams.add("classification", classification); + queryParams.add(QUERY, query); + queryParams.add("excludeDeletedEntities", String.valueOf(excludeDeletedEntities)); + queryParams.add(LIMIT, String.valueOf(limit)); + queryParams.add(OFFSET, String.valueOf(offset)); + queryParams.add("sortBy", "name"); + queryParams.add("sortOrder", String.valueOf(SortOrder.ASCENDING)); + + return callAPI(API_V2.BASIC_SEARCH, String.class, queryParams); + } + + /** + * 修改实体的注释 + */ + public String modifyComment(String guid,String commentStr) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("name", "comment"); + + return callAPI(API_V3.UPDATE_ENTITY_ATTR,String.class,commentStr,queryParams,guid); + } + + /** + * 设置实体的标签 + */ + public void setLabels(String guid, Set labels) throws AtlasServiceException { + API api = new API(String.format(SET_ENTITY_LABELS_BY_GUID_TEMPLATE, guid), HttpMethod.POST, Response.Status.NO_CONTENT); + callAPI(api, (Class)null, labels); + } + + /** + * 删除实体的标签 + */ + public void removeLabels(String guid, Set labels) throws AtlasServiceException { + API api = new API(String.format(DELETE_ENTITY_LABELS_BY_GUID_TEMPLATE, guid), HttpMethod.DELETE, Response.Status.NO_CONTENT); + callAPI(api, (Class)null, labels); + } + + /** + * 获取所有的类别 + */ + public String getClassificationDefForString() throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("type", "classification"); + return callAPI(API_V2.GET_ALL_TYPE_DEFS, String.class, queryParams); + } + + /** + * 根据名称获取分类 + */ + public String getClassificationDefByNameForString(String name) throws AtlasServiceException { + API api = new API(String.format(GET_CLASSIFICATION_BY_NAME_TEMPLATE, name), HttpMethod.GET, Response.Status.OK); + return callAPI(api, String.class, null); + } + + /** + * 根据名称删除类型 + */ + public void deleteTypedefByName(String name) throws AtlasServiceException { + callAPI(API_V3.DELETE_TYPEDEF_BY_NAME, (Class)null, null, name); + } + + /** + * 创建类型 + */ + public String createAtlasTypeDefsForString(final AtlasTypesDef typesDef) throws AtlasServiceException { + return callAPI(API_V2.CREATE_ALL_TYPE_DEFS, String.class, AtlasType.toJson(typesDef)); + } + + /** + * 更新类型 + */ + public void updateAtlasTypeDefsV2(final AtlasTypesDef typesDef) throws AtlasServiceException { + callAPI(API_V2.UPDATE_ALL_TYPE_DEFS, (Class)null, AtlasType.toJson(typesDef)); + } + + /** + * 为实体添加分类 + */ + @Override + public void addClassifications(String guid, List classifications) throws AtlasServiceException { + callAPI(formatPathParameters(API_V2.ADD_CLASSIFICATIONS, guid), (Class)null, AtlasType.toJson(classifications), (String[]) null); + } + + /** + * 为实体更新分类 + */ + @Override + public void updateClassifications(String guid, List classifications) throws AtlasServiceException { + callAPI(formatPathParameters(API_V2.UPDATE_CLASSIFICATIONS, guid), (Class)null, AtlasType.toJson(classifications)); + } + + /** + * 为实体删除分类 + */ + @Override + public void deleteClassifications(String guid, List classifications) throws AtlasServiceException { + if(classifications !=null && classifications.size() > 0) { + for (AtlasClassification c : classifications) { + this.deleteClassification(guid, c.getTypeName()); + } + } + } + + /** + * 查询实体的分类 + */ + public String getClassificationsForString(String guid) throws AtlasServiceException { + return callAPI(formatPathParameters(API_V2.GET_CLASSIFICATIONS, guid), String.class, null); + } + + + + + public static class API_V3 extends API { + public static final API_V3 GET_ENTITIES = new API_V3(ENTITIES_API, HttpMethod.GET, Response.Status.OK); + public static final API_V3 UPDATE_ENTITY_ATTR = new API_V3(UPDATE_ENTITY_ATTR_API,HttpMethod.PUT,Response.Status.OK); + public static final API_V3 DELETE_TYPEDEF_BY_NAME = new API_V3(TYPEDEF_BY_NAME, HttpMethod.DELETE, Response.Status.NO_CONTENT); + + private API_V3(String path, String method, Response.Status status) { + super(path, method, status); + } + } + + + private MultivaluedMap attributesToQueryParams(Map attributes) { + return attributesToQueryParams(attributes, null); + } + + private MultivaluedMap attributesToQueryParams(Map attributes, + MultivaluedMap queryParams) { + if (queryParams == null) { + queryParams = new MultivaluedMapImpl(); + } + + if (MapUtils.isNotEmpty(attributes)) { + for (Map.Entry e : attributes.entrySet()) { + queryParams.putSingle(PREFIX_ATTR + e.getKey(), e.getValue()); + } + } + + return queryParams; + } +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasService.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasService.java new file mode 100644 index 000000000..2eef11ce3 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasService.java @@ -0,0 +1,343 @@ +package com.webank.wedatasphere.dss.data.common.atlas; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.internal.LinkedTreeMap; +import com.webank.wedatasphere.dss.data.common.conf.AtlasConf; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Service; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service("atlasService") +public class AtlasService { + private AtlasClient atlasClient; + private final Gson gson; + + public AtlasService() { + String[] urls = new String[]{AtlasConf.ATLAS_REST_ADDRESS.getValue()}; + String[] basicAuthUsernamePassword = new String[]{AtlasConf.ATLAS_USERNAME.getValue(), AtlasConf.ATLAS_PASSWORD.getValue()}; + atlasClient = new AtlasClient(urls, basicAuthUsernamePassword); + + GsonBuilder builder = new GsonBuilder(); + // Register an adapter to manage the date types as long values + builder.registerTypeAdapter(Date.class, new JsonDeserializer() { + @Override + public Date deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + return new Date(json.getAsJsonPrimitive().getAsLong()); + } + }); + + gson = builder.create(); + } + + /** + * hive db数量 + */ + public long getHiveDbCnt() throws AtlasServiceException { + String jsonStr = atlasClient.getHiveDbs(); + if (StringUtils.isNotEmpty(jsonStr)) { + JsonObject jsonObject = gson.fromJson(jsonStr, JsonObject.class); + if (jsonObject != null) { + return jsonObject.get("count").getAsLong(); + } else { + return 0; + } + } else { + return 0; + } + } + + /** + * hive table数量 + */ + public long getHiveTableCnt() throws AtlasServiceException { + String jsonStr = atlasClient.getHiveTables(); + if (StringUtils.isNotEmpty(jsonStr)) { + JsonObject jsonObject = gson.fromJson(jsonStr, JsonObject.class); + if (jsonObject != null) { + return jsonObject.get("count").getAsLong(); + } else { + return 0; + } + } else { + return 0; + } + } + + + /** + * 根据关键字搜索hive table + */ + public List searchHiveTable(String classification, String query, + boolean excludeDeletedEntities, int limit, int offset) throws AtlasServiceException { + String jsonStr = atlasClient.basicSearchForString("hive_table", classification, query, excludeDeletedEntities, limit, offset); + AtlasSearchResult atlasSearchResult = gson.fromJson(jsonStr, AtlasSearchResult.class); + + return atlasSearchResult.getEntities(); + } + + /** + * 获取hive table对象 + */ + public AtlasEntity getHiveTblByGuid(String guid) throws AtlasServiceException { + String jsonStr = atlasClient.getEntityByGuidForString(guid, false, false); + + AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntityWithExtInfo.class); + + return atlasEntityWithExtInfo.getEntity(); + } + + /** + * 获取hive table对象 + */ + public AtlasEntity getHiveTblByAttribute(Map uniqAttributes, boolean minExtInfo, boolean ignoreRelationship) throws AtlasServiceException { + String jsonStr = atlasClient.getEntityByAttributeForString("hive_table", uniqAttributes, minExtInfo, ignoreRelationship); + + AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntityWithExtInfo.class); + + return atlasEntityWithExtInfo.getEntity(); + } + + /** + * 获取hive column对象 + */ + public AtlasEntity getHiveColumn(String guid) throws AtlasServiceException { + String jsonStr = atlasClient.getEntityByGuidForString(guid, true, true); + + AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntityWithExtInfo.class); + + return atlasEntityWithExtInfo.getEntity(); + } + + /** + * 获取多个hive column对象 + */ + public List getHiveColumnsByGuids(List guids) throws AtlasServiceException { + String jsonStr = atlasClient.getEntitiesByGuidsForString(guids, true, true); + AtlasEntity.AtlasEntitiesWithExtInfo atlasEntitiesWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntitiesWithExtInfo.class); + + return atlasEntitiesWithExtInfo.getEntities(); + } + + + /** + * 修改实体的注释 + */ + public void modifyComment(String guid, String commentStr) throws AtlasServiceException { + atlasClient.modifyComment(guid, commentStr); + } + + /** + * 设置实体的标签 + */ + public void setLabels(String guid, Set labels) throws AtlasServiceException { + atlasClient.setLabels(guid, labels); + } + + /** + * 删除实体的标签 + */ + public void removeLabels(String guid, Set labels) throws AtlasServiceException { + atlasClient.removeLabels(guid, labels); + } + + /** + * 血缘信息 + */ + public AtlasLineageInfo getLineageInfo(final String guid, final AtlasLineageInfo.LineageDirection direction, final int depth) throws AtlasServiceException { + String jsonStr = atlasClient.getLineageInfoForString(guid, direction, depth); + + AtlasLineageInfo atlasLineageInfo = gson.fromJson(jsonStr, AtlasLineageInfo.class); + + return atlasLineageInfo; + } + + /** + * 根据guid来获取hive tbl名称: db.table + */ + public String getHiveTblNameById(String guid) throws AtlasServiceException { + String jsonStr = atlasClient.getHeaderByIdForString(guid); + AtlasEntityHeader atlasEntityHeader = gson.fromJson(jsonStr, AtlasEntityHeader.class); + + return atlasEntityHeader.getAttribute("qualifiedName").toString().split("@")[0]; + } + + /** + * 根据guid来获取hive tbl名称 和 是否分区表、是否外部表、外部表路径 + */ + public Map getHiveTblAttributesByGuid(String guid) throws AtlasServiceException { + Map result = new HashMap<>(4); + + String jsonStr = atlasClient.getEntityByGuidForString(guid, false, false); + AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntityWithExtInfo.class); + + result.put("tblName", atlasEntityWithExtInfo.getEntity().getAttribute("qualifiedName").toString().split("@")[0]); + result.put("isPartition", ((List) atlasEntityWithExtInfo.getEntity().getAttribute("partitionKeys")).size() > 0); + result.put("tableType",atlasEntityWithExtInfo.getEntity().getAttribute("tableType")); + Map sdMap = (LinkedTreeMap)atlasEntityWithExtInfo.getEntity().getRelationshipAttribute("sd"); + if(null != sdMap) { + result.put("location",atlasEntityWithExtInfo.getReferredEntities().get(sdMap.get("guid")).getAttribute("location")); + } + return result; + } + + /** + * 获取所有的分类 + */ + public AtlasTypesDef getClassificationDef() throws AtlasServiceException { + String jsonStr = atlasClient.getClassificationDefForString(); + AtlasTypesDef atlasTypesDef = gson.fromJson(jsonStr, AtlasTypesDef.class); + + return atlasTypesDef; + } + + /** + * 根据名称获取分类 + */ + public AtlasClassificationDef getClassificationDefByName(String name) throws AtlasServiceException { + String jsonStr = atlasClient.getClassificationDefByNameForString(name); + AtlasClassificationDef atlasClassificationDef = gson.fromJson(jsonStr, AtlasClassificationDef.class); + + return atlasClassificationDef; + } + + /** + * 根据名称获取分类的一级子类型 + */ + public List getClassificationDefListByName(String name) throws AtlasServiceException { + List atlasClassificationDefList = new ArrayList(); + + AtlasClassificationDef atlasClassificationDef = this.getClassificationDefByName(name); + //atlasClassificationDefList.add(atlasClassificationDef); + + Set subTypes = atlasClassificationDef.getSubTypes(); + subTypes.forEach(item -> { + try { + atlasClassificationDefList.add(this.getClassificationDefByName(item)); + } catch (AtlasServiceException exception) { + exception.printStackTrace(); + } + + }); + + return atlasClassificationDefList; + } + + /** + * 获取所有分层的一级子类型,包括系统预置分层和用户自定义分层 + */ + public List getClassificationDefListForLayer() throws AtlasServiceException { + List atlasClassificationDefList = new ArrayList(); + // 系统预置分层的一级子分类 + atlasClassificationDefList.addAll(this.getClassificationDefListByName(AtlasConf.ATLAS_CLASSIFICATION_LAYER_SYSTEM.getValue())); + + // 用户自定义分层的一级子分类 + atlasClassificationDefList.addAll(this.getClassificationDefListByName(AtlasConf.ATLAS_CLASSIFICATION_LAYER_USER.getValue())); + + return atlasClassificationDefList; + } + + /** + * 创建新的类型定义 + */ + public AtlasTypesDef createAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasServiceException { + String jsonStr = atlasClient.createAtlasTypeDefsForString(typesDef); + AtlasTypesDef atlasTypesDef = gson.fromJson(jsonStr, AtlasTypesDef.class); + + return atlasTypesDef; + } + + /** + * 更新的类型定义 + */ + public void updateAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasServiceException { + atlasClient.updateAtlasTypeDefsV2(typesDef); + } + + /** + * 根据名称删除类型(几种类型:Enum-Struct-Classification-Entity-Relationship-BusinessMetadata) + */ + public void deleteTypedefByName(String name) throws AtlasServiceException { + atlasClient.deleteTypedefByName(name); + } + + /** + * 为实体添加分类 + */ + public void addClassifications(String guid, List classifications) throws AtlasServiceException { + atlasClient.addClassifications(guid, classifications); + } + + /** + * 为实体删除分类 + */ + public void deleteClassification(String guid, String classificationName) throws AtlasServiceException { + atlasClient.deleteClassification(guid, classificationName); + } + + /** + * 为实体删除分类 + */ + public void deleteClassifications(String guid, List classifications) throws AtlasServiceException { + atlasClient.deleteClassifications(guid, classifications); + } + + /** + * 为实体更新分类 + */ + public void updateClassifications(String guid, List classifications) throws AtlasServiceException { + atlasClient.updateClassifications(guid, classifications); + } + + /** + * 为实体删除已有的分类,添加新的分类 + */ + public void removeAndAddClassifications(String guid, List newClassifications) throws AtlasServiceException { + AtlasClassificationV2.AtlasClassificationsV2 oldAtlasClassifications = this.getClassifications(guid); + if (oldAtlasClassifications != null && oldAtlasClassifications.getList() != null) { + oldAtlasClassifications.getList().stream().forEach(ele -> { + try { + this.deleteClassification(guid, ele.getTypeName()); + } catch (AtlasServiceException exception) { + exception.printStackTrace(); + } + }); + } + if (newClassifications != null && newClassifications.size() > 0) { + atlasClient.addClassifications(guid, newClassifications); + } + } + + + /** + * 查询实体的分类 + */ + public AtlasClassificationV2.AtlasClassificationsV2 getClassifications(String guid) throws AtlasServiceException { + String jsonStr = atlasClient.getClassificationsForString(guid); + // org.codehaus.jackson.map.JsonMappingException: Conflicting getter definitions for property "propagate" + AtlasClassificationV2.AtlasClassificationsV2 atlasClassifications = gson.fromJson(jsonStr, AtlasClassificationV2.AtlasClassificationsV2.class); + + return atlasClassifications; + } +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/conf/AtlasConf.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/conf/AtlasConf.java new file mode 100644 index 000000000..7bfb0d0bd --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/conf/AtlasConf.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.data.common.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public interface AtlasConf { + /** + * atlas server info + */ + CommonVars ATLAS_APPLICATION_PROPERTIES = CommonVars.apply("atlas-application.properties"); + CommonVars ATLAS_REST_ADDRESS = CommonVars.apply("atlas.rest.address"); + CommonVars ATLAS_USERNAME = CommonVars.apply("atlas.username"); + CommonVars ATLAS_PASSWORD = CommonVars.apply("atlas.password"); + CommonVars ATLAS_CLUSTER_NAME = CommonVars.apply("atlas.cluster.name","primary"); + + /** + * 系统初始化分类信息: 主题域、系统分层、自定义分层 + */ + CommonVars ATLAS_CLASSIFICATION_SUBJECT = CommonVars.apply("atlas.classification.subject", "subject"); + CommonVars ATLAS_CLASSIFICATION_LAYER_SYSTEM = CommonVars.apply("atlas.classification.layer_system", "layer_system"); + CommonVars ATLAS_CLASSIFICATION_LAYER_USER = CommonVars.apply("atlas.classification.layer", "layer"); + + + /** + * hive metastore db + */ + CommonVars METASTORE_DATASOURCE_DRIVER = CommonVars.apply("metastore.datasource.driver", "com.mysql.jdbc.Driver"); + CommonVars METASTORE_DATASOURCE_URL = CommonVars.apply("metastore.datasource.url"); + CommonVars METASTORE_DATASOURCE_USERNAME = CommonVars.apply("metastore.datasource.username"); + CommonVars METASTORE_DATASOURCE_PASSWORD = CommonVars.apply("metastore.datasource.password"); + + +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/exception/DAOException.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/exception/DAOException.java new file mode 100644 index 000000000..529edafe3 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/exception/DAOException.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.data.common.exception; + + +public class DAOException extends RuntimeException { + public DAOException(){ + super(); + } + public DAOException (String message,Throwable cause){ + super(message,cause); + } + public DAOException(String message){ + super(message); + } + public DAOException(Throwable cause){ + super(cause); + } +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/exception/DataGovernanceException.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/exception/DataGovernanceException.java new file mode 100644 index 000000000..095c18196 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/exception/DataGovernanceException.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.dss.data.common.exception; + +public class DataGovernanceException extends RuntimeException { + + public DataGovernanceException(final String message) { + super(message); + } +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/DataSourceUtil.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/DataSourceUtil.java new file mode 100644 index 000000000..5794f74f9 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/DataSourceUtil.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.dss.data.common.utils; + +import com.alibaba.druid.pool.DruidDataSource; +import com.webank.wedatasphere.dss.data.common.conf.AtlasConf; + +public class DataSourceUtil { + private static DruidDataSource druidDataSource =null; + + private DataSourceUtil(){} + + public static DruidDataSource getDataSource(){ + if(druidDataSource ==null) { + synchronized (DataSourceUtil.class){ + if(druidDataSource ==null){ + druidDataSource = new DruidDataSource(); + druidDataSource.setDriverClassName(AtlasConf.METASTORE_DATASOURCE_DRIVER.getValue()); + druidDataSource.setUrl(AtlasConf.METASTORE_DATASOURCE_URL.getValue()); + druidDataSource.setUsername(AtlasConf.METASTORE_DATASOURCE_USERNAME.getValue()); + druidDataSource.setPassword(AtlasConf.METASTORE_DATASOURCE_PASSWORD.getValue()); + } + } + } + return druidDataSource; + } + +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/DateUtil.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/DateUtil.java new file mode 100644 index 000000000..c57c5b947 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/DateUtil.java @@ -0,0 +1,366 @@ +package com.webank.wedatasphere.dss.data.common.utils; + +import org.apache.atlas.SortOrder; + +import java.sql.Timestamp; +import java.text.DecimalFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +public class DateUtil { + + /** + * 英文简写(默认)如:2010-12-01 + */ + public static String FORMAT_MONTH = "yyyy-MM"; + /** + * 英文简写(默认)如:2010-12-01 + */ + public static String FORMAT_SHORT = "yyyy-MM-dd"; + + public static String FORMAT_SHORT_REPORT = "yyyy/MM/dd"; + /** + * 英文全称 如:2010-12-01 23:15:06 + */ + public static String FORMAT_LONG = "yyyy-MM-dd HH:mm:ss"; + public static String FORMAT_WITHOUT_SECOND = "yyyy-MM-dd HH:mm"; + + public static String FORMAT_MINUTE_REPORT = "yyyy/MM/dd HH:mm"; + public static String FORMAT_MINUTE = "yyyy-MM-dd HH:mm"; + /** + * 精确到毫秒的完整时间 如:yyyy-MM-dd HH:mm:ss.S + */ + public static String FORMAT_FULL = "yyyy-MM-dd HH:mm:ss.S"; + /** + * 中文简写 如:2010年12月01日 + */ + public static String FORMAT_SHORT_CN = "yyyy年MM月dd日"; + /** + * 中文全称 如:2010年12月01日 23时15分06秒 + */ + public static String FORMAT_LONG_CN = "yyyy年MM月dd日 HH时mm分ss秒"; + /** + * 精确到毫秒的完整中文时间 + */ + public static String FORMAT_FULL_CN = "yyyy年MM月dd日 HH时mm分ss秒SSS毫秒"; + + /** + * 获得默认的 date pattern + */ + public static String getDatePattern() { + return FORMAT_LONG; + } + + /** + * 根据预设格式返回当前日期 + * + * @return + */ + public static String getNow() { + return format(new Date()); + } + + /** + * 根据用户格式返回当前日期 + * + * @param format + * @return + */ + public static String getNow(String format) { + return format(new Date(), format); + } + + /** + * 使用预设格式格式化日期 + * + * @param date + * @return + */ + public static String format(Date date) { + return format(date, getDatePattern()); + } + + /** + * 使用用户格式格式化日期 + * + * @param date 日期 + * @param pattern 日期格式 + * @return + */ + public static String format(Date date, String pattern) { + String returnValue = ""; + if (date != null) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + returnValue = df.format(date); + } + return (returnValue); + } + + /** + * 使用预设格式提取字符串日期 + * + * @param strDate 日期字符串 + * @return + */ + public static Date parse(String strDate) { + return parse(strDate, getDatePattern()); + } + + /** + * 使用用户格式提取字符串日期 + * + * @param strDate 日期字符串 + * @param pattern 日期格式 + * @return + */ + public static Date parse(String strDate, String pattern) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + try { + return df.parse(strDate); + } catch (ParseException e) { + e.printStackTrace(); + return null; + } + } + + /** + * 在日期上增加数个整月 + * + * @param date 日期 + * @param n 要增加的月数 + * @return + */ + public static Date addMonth(Date date, int n) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.MONTH, n); + return cal.getTime(); + } + + /** + * 在日期上增加天数 + * + * @param date 日期 + * @param n 要增加的天数 + * @return + */ + public static Date addDay(Date date, int n) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.DATE, n); + return cal.getTime(); + } + + /** + * 获取时间戳 + */ + public static String getTimeString() { + SimpleDateFormat df = new SimpleDateFormat(FORMAT_FULL); + Calendar calendar = Calendar.getInstance(); + return df.format(calendar.getTime()); + } + + /** + * 获取日期年份 + * + * @param date 日期 + * @return + */ + public static String getYear(Date date) { + return format(date).substring(0, 4); + } + + /** + * 按默认格式的字符串距离今天的天数 + * + * @param date 日期字符串 + * @return + */ + public static int countDays(String date) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date)); + long t1 = c.getTime().getTime(); + return (int) (t / 1000 - t1 / 1000) / 3600 / 24; + } + + /** + * 按用户格式字符串距离今天的天数 + * + * @param date 日期字符串 + * @param format 日期格式 + * @return + */ + public static int countDaysAbs(String date, String format) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date, format)); + long t1 = c.getTime().getTime(); + return (int) Math.abs(t / 1000 - t1 / 1000) / 3600 / 24; + } + + /** + * 按用户格式字符串距离今天的天数 + * + * @param date 日期字符串 + * @param format 日期格式 + * @return + */ + public static int countDays(String date, String format) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date, format)); + long t1 = c.getTime().getTime(); + return (int) (t / 1000 - t1 / 1000) / 3600 / 24; + } + + public static String timeStrComplete(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return timeStr + lastStr; + } + return null; + } + + public static Timestamp str2Time(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return Timestamp.valueOf(timeStr + lastStr); + } + + return null; + } + + public static Timestamp str2TimeMax(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 23:59:59"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return Timestamp.valueOf(timeStr + lastStr); + } + + return null; + } + + public static Date str2DateTime(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return parse(timeStr + lastStr); + } + + return null; + } + + public static Date str2DateTimeMax(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 23:59:59"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return parse(timeStr + lastStr); + } + + return null; + } + + /** + * 字符串数字转化为double + * + * @param string + * @return + */ + public static double str2double(String string) { + if (null == string || "".equals(string)) { + return 0; + } + try { + return Double.valueOf(string); + } catch (NumberFormatException e) { + return 0; + } + } + + public static String addHours(Date date,int hours) { + SimpleDateFormat df = new SimpleDateFormat(FORMAT_LONG); + + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.HOUR, hours); + return df.format(cal.getTime()); + } + + public static Long strToTimeStamp(String date,String format) throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat(format); + return sdf.parse(date).getTime(); + } + + public static String unixToTimeStr(Long unixtime) { + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date (unixtime)); + } + + public static String unixToTimeStr(Double unixtime){ + String strTime =""; + if(unixtime !=null) { + strTime = new DecimalFormat("#").format(unixtime); + return unixToTimeStr(Long.valueOf(strTime)); + } + return null; + } +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/HttpClientUtil.java b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/HttpClientUtil.java new file mode 100644 index 000000000..e14fbe073 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/utils/HttpClientUtil.java @@ -0,0 +1,469 @@ +package com.webank.wedatasphere.dss.data.common.utils; + +import org.apache.http.Consts; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.SocketTimeoutException; +import java.net.URLEncoder; +import java.security.cert.CertificateException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; + + +public final class HttpClientUtil { + private final static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); + public final static int connectTimeout = 5000; + private static PoolingHttpClientConnectionManager connManager = null; + private static CloseableHttpClient httpclient = null; + + /** + * 重写验证方法,取消检测ssl + */ + private static TrustManager trustAllManager = new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + }; + + static { + httpclient = HttpClients.createDefault(); + } + + + /** + * + * @param url + * @param timeout + * @param headerMap + * @param paramsList + * @param encoding + * @return + */ + public static String postForm(String url, int timeout, Map headerMap, List paramsList, String encoding){ + HttpPost post = new HttpPost(url); + try { + if(headerMap != null){ + for(Entry entry : headerMap.entrySet()){ + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + //post.setHeader("Content-type", "application/json"); + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(timeout) + .setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + + post.setEntity(new UrlEncodedFormEntity(paramsList, encoding)); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if(entity != null){ + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if(entity != null){ + entity.getContent().close(); + } + } + } finally { + if(response != null){ + response.close(); + } + } + } catch (Exception e) { + throw new RuntimeException("invoke http post error!",e); + } finally { + post.releaseConnection(); + } + return ""; + } + + public static String postJsonBody(String url, int timeout, Map headerMap, + String paraData, String encoding) { + + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + StringEntity jsonEntity = new StringEntity(paraData, ContentType.APPLICATION_JSON); + post.setConfig(requestConfig); + post.setEntity(jsonEntity); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + @SuppressWarnings("deprecation") + public static String invokeGet(String url, Map params, String encode, int connectTimeout, + int soTimeout) { + String responseString = null; + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + + StringBuilder sb = new StringBuilder(); + sb.append(url); + int i = 0; + if (params != null) { + for (Entry entry : params.entrySet()) { + if (i == 0 && !url.contains("?")) { + sb.append("?"); + } else { + sb.append("&"); + } + sb.append(entry.getKey()); + sb.append("="); + String value = entry.getValue(); + try { + sb.append(URLEncoder.encode(value, "UTF-8")); + } catch (UnsupportedEncodingException e) { + logger.warn("encode http get params error, value is " + value, e); + sb.append(URLEncoder.encode(value)); + } + i++; + } + } + HttpGet get = new HttpGet(sb.toString()); + get.setConfig(requestConfig); + try { + CloseableHttpResponse response = httpclient.execute(get); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + responseString = EntityUtils.toString(entity, encode); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]get response error, url:%s", sb.toString()), e); + return responseString; + } finally { + if (response != null) { + response.close(); + } + } + // System.out.println(String.format("[HttpUtils Get]Debug url:%s , + // response string %s:", sb.toString(), responseString)); + } catch (SocketTimeoutException e) { + logger.error(String.format("[HttpUtils Get]invoke get timout error, url:%s", sb.toString()), e); + return responseString; + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]invoke get error, url:%s", sb.toString()), e); + } finally { + get.releaseConnection(); + } + return responseString; + } + + /** + * HTTPS请求,默认超时为5S + * + * @param reqURL + * @param params + * @return + */ + public static String connectPostHttps(String reqURL, Map params) { + + String responseContent = null; + HttpPost httpPost = new HttpPost(reqURL); + try { + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + List formParams = new ArrayList(); + httpPost.setEntity(new UrlEncodedFormEntity(formParams, Consts.UTF_8)); + httpPost.setConfig(requestConfig); + // 绑定到请求 Entry + for (Entry entry : params.entrySet()) { + formParams.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); + } + CloseableHttpResponse response = httpclient.execute(httpPost); + try { + // 执行POST请求 + HttpEntity entity = response.getEntity(); // 获取响应实体 + try { + if (null != entity) { + responseContent = EntityUtils.toString(entity, Consts.UTF_8); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + logger.info("requestURI : " + httpPost.getURI() + ", responseContent: " + responseContent); + } catch (ClientProtocolException e) { + logger.error("ClientProtocolException", e); + } catch (IOException e) { + logger.error("IOException", e); + } finally { + httpPost.releaseConnection(); + } + return responseContent; + + } + + class Test { + String v; + String k; + + public String getV() { + return v; + } + + public void setV(String v) { + this.v = v; + } + + public String getK() { + return k; + } + + public void setK(String k) { + this.k = k; + } + + } + + // 随机4位数 + public static String getRandomValue() { + String str = "0123456789"; + StringBuilder sb = new StringBuilder(4); + for (int i = 0; i < 4; i++) { + char ch = str.charAt(new Random().nextInt(str.length())); + sb.append(ch); + } + return sb.toString(); + } + + // 当前时间到秒 + public static String getTimestamp() { + + Date date = new Date(); + String timestamp = String.valueOf(date.getTime() / 1000); + return timestamp; + } + + // 当前时间到秒 + public static String getNowDate() { + Date date = new Date(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); + return sdf.format(date); + } + + public static String postJsonBody2(String url, int timeout, Map headerMap, + List paramsList, String encoding) { + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(paramsList, encoding); + post.setEntity(entity); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + public static String postJsonBody3(String url, int timeout, Map headerMap, + Map paramsList, String encoding) { + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + //JSONArray jsonArray = JSONArray.fromObject(paramsList); + //post.setEntity(new StringEntity(jsonArray.get(0).toString(), encoding)); + post.setEntity(new StringEntity(null, encoding)); + //logger.info("successfully start post Json Body url{},params ", url,jsonArray.get(0).toString()); + logger.info("successfully start post Json Body url{},params ", url,null); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + public static String executeGet(String url) + { + String rtnStr = ""; + HttpGet httpGet = new HttpGet(url); + try { + HttpResponse httpResponse = httpclient.execute(httpGet); + //获得返回的结果 + rtnStr = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + e.printStackTrace(); + } finally { + httpGet.releaseConnection(); + } + return rtnStr; + } + + + public static String executeGetByHeader(String url,Map headerMap) + { + String rtnStr = ""; + HttpGet httpGet = new HttpGet(url); + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + httpGet.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + try { + HttpResponse httpResponse = httpclient.execute(httpGet); + System.out.println(httpResponse.getStatusLine().getStatusCode()); + //获得返回的结果 + rtnStr = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + e.printStackTrace(); + } finally { + httpGet.releaseConnection(); + } + return rtnStr; + } + +} diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml b/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml new file mode 100644 index 000000000..6451c5c60 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml @@ -0,0 +1,126 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-data-governance-server + + + 2.1.0 + + + + com.webank.wedatasphere.dss + dss-data-asset-server + ${dss.version} + + + com.webank.wedatasphere.dss + dss-data-classification-server + ${dss.version} + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + org.springframework.cloud + spring-cloud-netflix + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/assembly/distribution.xml b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..486f36321 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/assembly/distribution.xml @@ -0,0 +1,46 @@ + + + + dss-data-governance-server + + dir + + true + dss-data-governance-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/application-dss.yml b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/application-dss.yml new file mode 100644 index 000000000..b0473838d --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/application-dss.yml @@ -0,0 +1,23 @@ + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + #instance: + #prefer-ip-address: true + #instance-id: ${spring.cloud.client.ip-address}:${server.port} + #metadata-map: + #test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/atlas-application.properties b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/atlas-application.properties new file mode 100644 index 000000000..e69de29bb diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss-data-governance-server.properties b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss-data-governance-server.properties new file mode 100644 index 000000000..b3980c6cc --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss-data-governance-server.properties @@ -0,0 +1,53 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9209 +spring.spring.application.name=dss-data-governance-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.asset.restful,com.webank.wedatasphere.dss.data.classification.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/asset/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.asset.entity +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.asset.dao,com.webank.wedatasphere.dss.data.warehouse.dao,com.webank.wedatasphere.dss.data.warehouse.mapper + +#wds.linkis.gateway.ip=127.0.0.1 +#wds.linkis.gateway.port=9001 +#wds.linkis.gateway.url=http://127.0.0.1:9001/ + + +# atlas config +atlas.rest.address=http://xxxxxxx:21000 +atlas.username=xxxxxxxx +atlas.password=yyyyyyyyy +atlas.client.readTimeoutMSecs=60000 +atlas.client.connectTimeoutMSecs=60000 + +atlas.cluster.name=primary + +# hive metadata config +metastore.datasource.driver=com.mysql.jdbc.Driver +metastore.datasource.url=jdbc:mysql://xxxxxx:yyyy/metastore?characterEncoding=UTF-8 +metastore.datasource.username=xxxxxx +metastore.datasource.password=yyyyyy \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss.properties b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss.properties new file mode 100644 index 000000000..41a8e2820 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss.properties @@ -0,0 +1,25 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +wds.linkis.gateway.ip=127.0.0.1 +wds.linkis.gateway.port=9001 +wds.linkis.gateway.url=http://127.0.0.1:9001/ + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://xxxxxxx:3306/dss_test?characterEncoding=UTF-8 +wds.linkis.server.mybatis.datasource.username=xxxxx +wds.linkis.server.mybatis.datasource.password=xxxxxx diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/log4j.properties b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/log4j.properties new file mode 100644 index 000000000..f88f949da --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/log4j.properties @@ -0,0 +1,38 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/dss-data-governance-server.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/log4j2.xml b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/log4j2.xml new file mode 100644 index 000000000..c00e1eaa7 --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/log4j2.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/token.properties b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/token.properties new file mode 100644 index 000000000..a52a8983d --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/token.properties @@ -0,0 +1,19 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +${userName}=${password} \ No newline at end of file diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DSSDataGovernanceApplication.scala b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DSSDataGovernanceApplication.scala new file mode 100644 index 000000000..1fe794bff --- /dev/null +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DSSDataGovernanceApplication.scala @@ -0,0 +1,41 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.governance + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + + +object DSSDataGovernanceApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-apps/dss-data-governance/pom.xml b/dss-apps/dss-data-governance/pom.xml new file mode 100644 index 000000000..e63bff2d2 --- /dev/null +++ b/dss-apps/dss-data-governance/pom.xml @@ -0,0 +1,22 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../pom.xml + + 4.0.0 + + dss-data-governance + pom + + dss-data-governance-server + dss-data-governance-common + dss-data-classification-server + dss-data-asset-server + + + \ No newline at end of file diff --git a/dss-apps/dss-scriptis-server/pom.xml b/dss-apps/dss-scriptis-server/pom.xml new file mode 100644 index 000000000..f1b411b1c --- /dev/null +++ b/dss-apps/dss-scriptis-server/pom.xml @@ -0,0 +1,283 @@ + + + + + 4.0.0 + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../pom.xml + + + dss-scriptis-server + + + UTF-8 + 2.16 + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + asm + org.ow2.asm + + + hk2-api + org.glassfish.hk2 + + + jersey-common + org.glassfish.jersey.core + + + linkis-common + org.apache.linkis + + + + + com.alibaba + fastjson + 1.2.83 + + + + org.glassfish.jersey.ext + jersey-bean-validation + ${jersey.version} + provided + + + javax.ws.rs-api + javax.ws.rs + + + hk2-locator + org.glassfish.hk2 + + + hk2-api + org.glassfish.hk2 + + + hibernate-validator + org.hibernate + + + jersey-server + org.glassfish.jersey.core + + + + + + org.apache.linkis + linkis-bml-client + ${linkis.version} + + + commons-beanutils + commons-beanutils + + + linkis-common + org.apache.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + org.apache.linkis + linkis-computation-client + ${linkis.version} + + + commons-beanutils + commons-beanutils + + + linkis-common + org.apache.linkis + + + + + org.postgresql + postgresql + 42.3.8 + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + org.apache.linkis + linkis-storage + ${linkis.version} + provided + + + linkis-common + org.apache.linkis + + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + archaius-core + com.netflix.archaius + + + slf4j-api + org.slf4j + + + spring-cloud-starter + org.springframework.cloud + + + spring-web + org.springframework + + + + + hk2-api + org.glassfish.hk2 + 2.4.0-b34 + + + org.modelmapper + modelmapper + 0.7.5 + + + + io.jsonwebtoken + jjwt + 0.6.0 + + + jackson-databind + com.fasterxml.jackson.core + + + + + + junit + junit + 4.13.1 + test + + + org.junit.platform + junit-platform-launcher + 1.5.2 + test + + + xstream + com.thoughtworks.xstream + 1.4.19 + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + ${basedir}/src/main/resources + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/dss-apps/dss-scriptis-server/src/main/assembly/distribution.xml b/dss-apps/dss-scriptis-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..33c52b17f --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/assembly/distribution.xml @@ -0,0 +1,44 @@ + + + + dss-apiService-server + + dir + + true + dss-scriptis-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/config/DSSScriptisConfiguration.java b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/config/DSSScriptisConfiguration.java new file mode 100644 index 000000000..86bae03e1 --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/config/DSSScriptisConfiguration.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis.config; + +import org.apache.linkis.common.conf.CommonVars; + + +public class DSSScriptisConfiguration { + public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.api.service.author.user.token","WS-AUTH"); + public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.api.service.adminuser","ws"); + + public final static CommonVars LINKIS_CONNECTION_TIMEOUT = CommonVars.apply("wds.linkis.client.connection.timeout",45000L); + public final static CommonVars LINKIS_READ_TIMEOUT = CommonVars.apply("wds.linkis.client.read.timeout",45000L); + + public final static String GLOBAL_LIMITS_PREFIX = "wds.dss.scriptis.global.limits."; + public final static String GLOBAL_LIMIT_PREFIX = "wds.dss.scriptis.global.limit."; + +} diff --git a/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/execute/LinkisJobSubmit.java b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/execute/LinkisJobSubmit.java new file mode 100644 index 000000000..54f440052 --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/execute/LinkisJobSubmit.java @@ -0,0 +1,101 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis.execute; + +import com.webank.wedatasphere.dss.scriptis.config.DSSScriptisConfiguration; +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.apache.linkis.server.conf.ServerConfiguration; +import org.apache.linkis.server.socket.controller.ServerEvent; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.UJESClientImpl; +import org.apache.linkis.ujes.client.request.JobExecuteAction; +import org.apache.linkis.ujes.client.response.JobExecuteResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + + +public class LinkisJobSubmit { + + private static UJESClient ujesClient; + + public static UJESClient getClient() { + if(ujesClient == null) { + synchronized (LinkisJobSubmit.class) { + if(ujesClient == null) { + ujesClient = getUJESClient( + Configuration.GATEWAY_URL().getValue(), + DSSScriptisConfiguration.LINKIS_ADMIN_USER.getValue(), + DSSScriptisConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue()); + } + } + } + return ujesClient; + } + + public static UJESClient getUJESClient(String url, String user, String token){ + return new UJESClientImpl(getClientConfig(url, user, token)); + } + + public static DWSClientConfig getClientConfig(String url, String user, String token){ + return ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() + .addServerUrl(url) + .connectionTimeout(DSSScriptisConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue()) + .discoveryEnabled(false).discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(5) + .retryEnabled(false).readTimeout(DSSScriptisConfiguration.LINKIS_READ_TIMEOUT.getValue()) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) + .setAuthTokenKey(user).setAuthTokenValue(token))) + .setDWSVersion(ServerConfiguration.BDP_SERVER_VERSION()).build(); + } + + /** + * @param operation + * @return + */ + public static JobExecuteResult execute(ServerEvent operation) { + Map dataMap = operation.getData(); + String runType = (String) dataMap.get("runType"); + String background = (String) dataMap.get("background"); + String executionCode = (String) dataMap.get("executionCode"); + String scriptPath = "default.scala"; + Map sourceMap = (Map) dataMap.get("source"); + scriptPath = sourceMap != null && sourceMap.containsKey("scriptPath") ? sourceMap.get("scriptPath") : scriptPath; + + String executeApplicationName = (String)dataMap.get("executeApplicationName"); + String umUser = (String)dataMap.get("umUser"); + Map paramsMap = (Map) dataMap.get("params"); + + Map source = new HashMap<>(1); + source.put("DSS-Scriptis", scriptPath); + JobExecuteAction.Builder builder = JobExecuteAction.builder().setCreator("IDE") + .addExecuteCode(executionCode) + .setEngineTypeStr(executeApplicationName) + .setRunTypeStr(runType) + .setUser(umUser) + .setParams(paramsMap) + .setSource(source); + JobExecuteAction jobAction = builder.build(); + return getClient().execute(jobAction); + } + +} diff --git a/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisAuthRestfulApi.java b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisAuthRestfulApi.java new file mode 100644 index 000000000..30efdf73b --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisAuthRestfulApi.java @@ -0,0 +1,76 @@ +package com.webank.wedatasphere.dss.scriptis.restful; + +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.BDPConfiguration; +import org.apache.linkis.protocol.util.ImmutablePair; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +import static com.webank.wedatasphere.dss.scriptis.config.DSSScriptisConfiguration.GLOBAL_LIMITS_PREFIX; +import static com.webank.wedatasphere.dss.scriptis.config.DSSScriptisConfiguration.GLOBAL_LIMIT_PREFIX; + +/** + * @author enjoyyin + * @date 2022-03-29 + * @since 0.5.0 + */ +@RequestMapping(path = "/dss/scriptis", produces = {"application/json"}) +@RestController +public class ScriptisAuthRestfulApi { + + private final Logger logger = LoggerFactory.getLogger(ScriptisAuthRestfulApi.class); + private Map globalLimits; + private final Map> globalLimitContents = new HashMap<>(); + + @RequestMapping(value = "/globalLimits",method = RequestMethod.GET) + public Message globalLimits() { + if(globalLimits == null) { + synchronized (ScriptisAuthRestfulApi.class) { + if(globalLimits == null) { + globalLimits = getMap(GLOBAL_LIMITS_PREFIX); + logger.info("loaded Scriptis global limits is {}.", globalLimits); + } + } + } + return Message.ok().data("globalLimits", globalLimits); + } + + @RequestMapping(value = "/globalLimits/{globalLimitName}",method = RequestMethod.GET) + public Message globalLimit(@PathVariable("globalLimitName") String globalLimitName) { + Supplier function = () -> Message.ok().data("globalLimitName", globalLimitName).data("content", globalLimitContents.get(globalLimitName)); + if(globalLimitContents.containsKey(globalLimitName)) { + return function.get(); + } + synchronized (globalLimitContents) { + if(!globalLimitContents.containsKey(globalLimitName)) { + Map globalLimitContent = getMap(GLOBAL_LIMIT_PREFIX + globalLimitName + "."); + logger.info("loaded Scriptis global limit {} content are {}.", globalLimitName, globalLimitContent); + globalLimitContents.put(globalLimitName, globalLimitContent); + } + } + return function.get(); + } + + private Map getMap(String prefix) { + return BDPConfiguration.properties().entrySet().stream() + .filter(entry -> entry.getKey().toString().startsWith(prefix) && entry.getValue() != null && StringUtils.isNotBlank(entry.getValue().toString())) + .map(entry -> { + String key = ((String) entry.getKey()).substring(prefix.length()); + if("true".equals(entry.getValue()) || "false".equals(entry.getValue())) { + return new ImmutablePair<>(key, Boolean.parseBoolean((String) entry.getValue())); + } else { + return new ImmutablePair<>(key, entry.getValue()); + } + }).collect(HashMap::new, (map, pair) -> map.put(pair.getKey(), pair.getValue()), + HashMap::putAll); + } +} diff --git a/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisBackgroundServiceRestfulApi.java b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisBackgroundServiceRestfulApi.java new file mode 100644 index 000000000..d8e75464e --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisBackgroundServiceRestfulApi.java @@ -0,0 +1,89 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis.restful; + +import com.google.gson.internal.LinkedTreeMap; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.scriptis.entrance.background.BackGroundService; +import com.webank.wedatasphere.dss.scriptis.entrance.background.ExportBackGroundService; +import com.webank.wedatasphere.dss.scriptis.entrance.background.LoadBackGroundService; +import com.webank.wedatasphere.dss.scriptis.execute.LinkisJobSubmit; +import org.apache.linkis.protocol.constants.TaskConstant; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.apache.linkis.server.socket.controller.ServerEvent; +import org.apache.linkis.server.utils.ModuleUserUtils; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import java.util.Map; + + +@RequestMapping(path = "/dss/scriptis", produces = {"application/json"}) +@RestController +public class ScriptisBackgroundServiceRestfulApi { + + private final Logger logger = LoggerFactory.getLogger(ScriptisBackgroundServiceRestfulApi.class); + + @RequestMapping(value = "/backgroundservice", method = RequestMethod.POST) + public Message backgroundService(HttpServletRequest req, @RequestBody Map json) { + String user = SecurityFilter.getLoginUsername(req); + String backgroundType = (String) json.get("background"); + logger.info("Begin to submit a '{}' background job for user {}.", backgroundType, user); + BackGroundService bgService; + if("export".equals(backgroundType)){ + bgService = new ExportBackGroundService(); + }else if("load".equals(backgroundType)){ + bgService = new LoadBackGroundService(); + }else{ + return Message.error("export type is not exist:"+backgroundType); + } + Map executionCode = (Map) json.get("executionCode"); + executionCode = DSSCommonUtils.COMMON_GSON.fromJson(DSSCommonUtils.COMMON_GSON.toJson(executionCode), LinkedTreeMap.class); + String username = ModuleUserUtils.getOperationUser(req, "backgroundservice"); + json.put("executionCode", executionCode); + json.put(TaskConstant.UMUSER, username); + ServerEvent serverEvent = new ServerEvent(); + serverEvent.setData(json); + serverEvent.setUser(username); + ServerEvent operation = bgService.operation(serverEvent); + JobExecuteResult jobExecuteResult = toLinkisEntrance(operation); + logger.info("submitted background job with execID: {}, taskID: {}.", jobExecuteResult.getExecID(), jobExecuteResult.getTaskID()); + return Message.ok().data("operation", operation) + .data("execID", jobExecuteResult.getExecID()) + .data("taskID", jobExecuteResult.getTaskID()); + } + + public JobExecuteResult toLinkisEntrance(ServerEvent operation){ + JobExecuteResult jobExecuteResult; + try{ + jobExecuteResult = LinkisJobSubmit.execute(operation); + }catch (Exception e){ + logger.error("submit background job failed.", e); + throw e; + } + return jobExecuteResult; + + } +} diff --git a/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/DSSScriptisServerApplication.scala b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/DSSScriptisServerApplication.scala new file mode 100644 index 000000000..f317906e1 --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/DSSScriptisServerApplication.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + + +object DSSScriptisServerApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/AbstractBackGroundService.scala b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/AbstractBackGroundService.scala new file mode 100644 index 000000000..5b1dc371c --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/AbstractBackGroundService.scala @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis.entrance.background + + +abstract class AbstractBackGroundService extends BackGroundService{ + +} diff --git a/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/BackGroundService.scala b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/BackGroundService.scala new file mode 100644 index 000000000..168f907c4 --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/BackGroundService.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis.entrance.background + +import org.apache.linkis.server.socket.controller.ServerEvent + + +trait BackGroundService { + val serviceType:String + def operation(serverEvent: ServerEvent):ServerEvent +} diff --git a/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/BackGroundServiceUtils.scala b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/BackGroundServiceUtils.scala new file mode 100644 index 000000000..9fa070de3 --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/BackGroundServiceUtils.scala @@ -0,0 +1,113 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis.entrance.background + +import java.io.{InputStream, OutputStream} +import java.lang +import java.lang.reflect.Type +import java.text.SimpleDateFormat +import java.util.Date + +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.gson.{GsonBuilder, JsonElement, JsonPrimitive, JsonSerializationContext, JsonSerializer} +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.io.FsPath +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.utils.FileSystemUtils +import org.apache.commons.io.IOUtils +import org.apache.commons.lang.time.DateFormatUtils + +object BackGroundServiceUtils extends Logging { + + private val CODE_STORE_PREFIX = CommonVars("bdp.dataworkcloud.bgservice.store.prefix", "hdfs:///tmp/bdp-ide/") + private val CODE_STORE_SUFFIX = CommonVars("bdp.dataworkcloud.bgservice.store.suffix", "") + private val CHARSET = "utf-8" + private val CODE_SPLIT = ";" + private val LENGTH_SPLIT = "#" + + def storeExecutionCode(destination: String,user:String): String = { + if (destination.length < 60000) return null + val path: String = getCodeStorePath(user) + val fsPath: FsPath = new FsPath(path) + val fileSystem = FSFactory.getFsByProxyUser(fsPath, user).asInstanceOf[FileSystem] + fileSystem.init(null) + var os: OutputStream = null + var position = 0L + val codeBytes = destination.getBytes(CHARSET) + Utils.tryFinally { + path.intern() synchronized { + if (!fileSystem.exists(fsPath)) FileSystemUtils.createNewFile(fsPath, user, true) + os = fileSystem.write(fsPath, false) + position = fileSystem.get(path).getLength + IOUtils.write(codeBytes, os) + } + } { + if (fileSystem != null) fileSystem.close() + IOUtils.closeQuietly(os) + } + val length = codeBytes.length + path + CODE_SPLIT + position + LENGTH_SPLIT + length + } + + def exchangeExecutionCode(codePath: String): Unit = { + import scala.util.control.Breaks._ + val path = codePath.substring(0, codePath.lastIndexOf(CODE_SPLIT)) + val codeInfo = codePath.substring(codePath.lastIndexOf(CODE_SPLIT) + 1) + val infos: Array[String] = codeInfo.split(LENGTH_SPLIT) + val position = infos(0).toInt + var lengthLeft = infos(1).toInt + val tub = new Array[Byte](1024) + val executionCode: StringBuilder = new StringBuilder + val fsPath: FsPath = new FsPath(path) + val fileSystem = FSFactory.getFsByProxyUser(fsPath, System.getProperty("user.name")).asInstanceOf[FileSystem] + fileSystem.init(null) + var is: InputStream = null + if(!fileSystem.exists(fsPath)) return + Utils.tryFinally { + is = fileSystem.read(fsPath) + if (position > 0) is.skip(position) + breakable { + while (lengthLeft > 0) { + val readed = is.read(tub) + val useful = Math.min(readed, lengthLeft) + if (useful < 0) break() + lengthLeft -= useful + executionCode.append(new String(tub, 0, useful, CHARSET)) + } + } + } { + if (fileSystem != null) fileSystem.close() + IOUtils.closeQuietly(is) + } + executionCode.toString() + } + + private def getCodeStorePath(user: String): String = { + val date: String = DateFormatUtils.format(new Date, "yyyyMMdd") + s"${CODE_STORE_PREFIX.getValue}${user}${CODE_STORE_SUFFIX.getValue}/executionCode/${date}/_bgservice" + } + + implicit val gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls + .registerTypeAdapter(classOf[java.lang.Double], new JsonSerializer[java.lang.Double] { + override def serialize(t: lang.Double, `type`: Type, jsonSerializationContext: JsonSerializationContext): JsonElement = + if(t == t.longValue()) new JsonPrimitive(t.longValue()) else new JsonPrimitive(t) + }).create + + implicit val jacksonJson = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) +} diff --git a/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/ExportBackGroundService.scala b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/ExportBackGroundService.scala new file mode 100644 index 000000000..88c2b0ef2 --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/ExportBackGroundService.scala @@ -0,0 +1,101 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis.entrance.background + +import java.util + +import com.google.gson.internal.LinkedTreeMap +import com.google.gson.{JsonObject, JsonParser} +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.server._ +import org.apache.linkis.server.socket.controller.ServerEvent + + +class ExportBackGroundService extends AbstractBackGroundService with Logging{ + override val serviceType: String = "export" + + override def operation(serverEvent: ServerEvent): ServerEvent = { + val params = serverEvent.getData.map { case (k, v) => k -> v.asInstanceOf[Any] } + //val executionCode = params.get("executionCode").get + val ec = params.get("executionCode").get.asInstanceOf[LinkedTreeMap[String,LinkedTreeMap[String,String]]] + if(ec.get("destination")!=null && ec.get("destination").get("fieldDelimiter") != null){ + info(s"---${ec.get("destination").get("fieldDelimiter")}---") + ec.get("destination").get("fieldDelimiter") match { + case "\\t" =>ec.get("destination").put("fieldDelimiter","\t") + case _ =>info("---other fieldDelimiter---") + } + } + val executionCode = BackGroundServiceUtils.gson.toJson(params.get("executionCode").get) + // TODO: Head may be removed + var newExecutionCode = "" + val jsonParser = new JsonParser() + val jsonCode = jsonParser.parse(executionCode.asInstanceOf[String]).asInstanceOf[JsonObject] + val destination = "val destination = \"\"\"" + jsonCode.get("destination").toString + "\"\"\"\n" + val dataInfo = jsonCode.get("dataInfo").toString + var newDataInfo = "val dataInfo = \"\"\"" + val storePath = BackGroundServiceUtils.storeExecutionCode(dataInfo,serverEvent.getUser) + if(storePath == null) newDataInfo += dataInfo + "\"\"\"\n" else newDataInfo += storePath + "\"\"\"\n" + newExecutionCode += destination + newExecutionCode += newDataInfo + if(storePath == null) + newExecutionCode += "org.apache.linkis.engineplugin.spark.imexport.ExportData.exportData(spark,dataInfo,destination)" + else + newExecutionCode += "org.apache.linkis.engineplugin.spark.imexport.ExportData.exportDataByFile(spark,dataInfo,destination)" + params.put("executionCode", newExecutionCode) + print(newExecutionCode) + val map = new util.HashMap[String, Object]() + params.foreach(f => map.put(f._1, f._2.asInstanceOf[Object])) + serverEvent.setData(map) + serverEvent + } + + def splitDataInfo(dataInfo:String):util.ArrayList[String] = { + val length = 6000 + val list = new util.ArrayList[String]() + var size = dataInfo.length /length + if(dataInfo.length % length != 0) size += 1 + for(i <- 0 to size-1){ + list.add(subString(dataInfo,i * length,(i +1) * length)) + } + list + } + + private def subString(str:String,begin:Int,end:Int):String = { + if(begin > str.length) return null + if(end > str.length) return str.substring(begin,str.length) + str.substring(begin,end) + } + + +} + +object A{ + def main(args: Array[String]): Unit = { + val builder: StringBuilder = new StringBuilder + for(i <- 1 to 200){ + if(i == 1) builder.append(2) else builder.append(1) + + } + var newDataInfo = "val dataInfo = \"\"\"" + val service: ExportBackGroundService = new ExportBackGroundService() + val splitDataInfos: util.ArrayList[String] =service.splitDataInfo(builder.toString()) + val splitString = "\"\"\"" + "+" + "\"\"\"" + val compaction = splitDataInfos.foldLeft("")((l,r) => l + splitString + r ) + newDataInfo += compaction.substring(splitString.length,compaction.length) + "\"\"\"\n" + print(newDataInfo) + } +} diff --git a/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/LoadBackGroundService.scala b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/LoadBackGroundService.scala new file mode 100644 index 000000000..efb85192c --- /dev/null +++ b/dss-apps/dss-scriptis-server/src/main/scala/com/webank/wedatasphere/dss/scriptis/entrance/background/LoadBackGroundService.scala @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.scriptis.entrance.background + +import java.util + +import com.google.gson.{JsonObject, JsonParser} +import org.apache.linkis.server._ +import org.apache.linkis.server.socket.controller.ServerEvent + + +class LoadBackGroundService extends AbstractBackGroundService { + + override val serviceType: String = "load" + + override def operation(serverEvent: ServerEvent): ServerEvent = { + val params = serverEvent.getData.map { case (k, v) => k -> v.asInstanceOf[Any] } + val executionCode = BackGroundServiceUtils.gson.toJson(params.get("executionCode").get) + // TODO: Head may be removed + var newExecutionCode = "" + val jsonParser = new JsonParser() + val jsonCode = jsonParser.parse(executionCode.asInstanceOf[String]).asInstanceOf[JsonObject] + val source = "val source = \"\"\"" + jsonCode.get("source").toString + "\"\"\"\n" + val destination = jsonCode.get("destination").toString + var newDestination = "val destination = \"\"\"" + val storePath = BackGroundServiceUtils.storeExecutionCode(destination,serverEvent.getUser) + if(storePath == null) newDestination +=destination +"\"\"\"\n" else newDestination +=storePath + "\"\"\"\n" + newExecutionCode += source + newExecutionCode += newDestination + if(storePath == null){ + newExecutionCode += "org.apache.linkis.engineplugin.spark.imexport.LoadData.loadDataToTable(spark,source,destination)" + }else{ + newExecutionCode += "org.apache.linkis.engineplugin.spark.imexport.LoadData.loadDataToTableByFile(spark,destination,source)" + } + + params.put("executionCode", newExecutionCode) + print(newExecutionCode) + val map = new util.HashMap[String, Object]() + params.foreach(f => map.put(f._1, f._2.asInstanceOf[Object])) + serverEvent.setData(map) + serverEvent + } + + def splitDestination(destination:String):util.ArrayList[String] = { + val length = 6000 + val list = new util.ArrayList[String]() + var size = destination.length /length + if(destination.length % length != 0) size += 1 + for(i <- 0 to size-1){ + list.add(subString(destination,i * length,(i +1) * length)) + } + list + } + + private def subString(str:String,begin:Int,end:Int):String = { + if(begin > str.length) return null + if(end > str.length) return str.substring(begin,str.length) + str.substring(begin,end) + } +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml b/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml new file mode 100644 index 000000000..34b8e7d9c --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml @@ -0,0 +1,127 @@ + + + + dss-user-guide + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-user-guide-server + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + org.springframework.cloud + spring-cloud-netflix + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + org.projectlombok + lombok + 1.18.16 + compile + + + com.vladsch.flexmark + flexmark + 0.62.2 + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/assembly/distribution.xml b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..c056c0683 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/assembly/distribution.xml @@ -0,0 +1,46 @@ + + + + dss-guide-server + + dir + + true + dss-user-guide-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/conf/GuideConf.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/conf/GuideConf.java new file mode 100644 index 000000000..e3c0e111f --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/conf/GuideConf.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.guide.server.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public interface GuideConf { + CommonVars GUIDE_CONTENT_IMAGES_PATH = CommonVars.apply("guide.content.images.path", "/usr/local/anlexander/all_bak/dss_linkis/dss-linkis-1.0.2/images"); + + CommonVars GUIDE_CHAPTER_IMAGES_PATH = CommonVars.apply("guide.chapter.images.path", "/usr/local/anlexander/all_bak/dss_linkis/dss-linkis-1.0.2/images"); + + /**gitbook路径*/ + CommonVars HOST_GITBOOK_PATH = CommonVars.apply("host.gitbook.path", "/appcom/Install/ApacheInstall/gitbook_books"); + + CommonVars HOST_IP_ADDRESS = CommonVars.apply("target.ip.address", "127.0.0.1"); + + CommonVars TARGET_GITBOOK_PATH = CommonVars.apply("target.gitbook.path", "/appcom/Install/ApacheInstall"); + + CommonVars SUMMARY_IGNORE_MODEL = CommonVars.apply("summary.ignore.model","km"); + + CommonVars GUIDE_SYNC_MODEL = CommonVars.apply("guide.sync.model","gitbook"); + +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideCatalogMapper.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideCatalogMapper.java new file mode 100644 index 000000000..6d3df643d --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideCatalogMapper.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.guide.server.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.guide.server.entity.GuideCatalog; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +@Mapper +public interface GuideCatalogMapper extends BaseMapper { + /** + * parent_id =-1 标识该目录属于最顶层的一级目录 + */ + @Select("SELECT * FROM dss_guide_catalog WHERE is_delete =0 AND parent_id =-1 ORDER BY id ASC") + List queryGuideCatalogListForTop(); + + @Select("SELECT * FROM dss_guide_catalog WHERE is_delete =0 AND parent_id =#{id} ORDER BY id ASC") + List queryGuideCatalogChildrenById(@Param("id") Long id); + + @Insert({ + "" + }) + int batchInsert(@Param("list") List list); +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideChapterMapper.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideChapterMapper.java new file mode 100644 index 000000000..397a76201 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideChapterMapper.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.guide.server.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.guide.server.entity.GuideChapter; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import javax.ws.rs.QueryParam; +import java.util.List; + +@Mapper +public interface GuideChapterMapper extends BaseMapper { + @Select("SELECT * FROM dss_guide_chapter WHERE is_delete =0 AND catalog_id =#{catalogId} ORDER BY id ASC") + List queryGuideChapterListByCatalogId(@Param("catalogId") Long catalogId); + + @Select("SELECT * FROM dss_guide_chapter WHERE is_delete =0 AND (content LIKE CONCAT('%', #{keyword}, '%') OR title LIKE CONCAT('%', #{keyword}, '%')) ORDER BY id ASC") + List searchGuideChapterListByKeyword(@Param("keyword") String keyword); + + int batchInsert(@Param("list") List list); + + @Select("truncate table dss_guide_chapter;") + void initChapterId(); +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideContentMapper.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideContentMapper.java new file mode 100644 index 000000000..2439f9d1b --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideContentMapper.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.guide.server.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.guide.server.entity.GuideContent; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +import java.util.List; + +@Mapper +public interface GuideContentMapper extends BaseMapper { + @Select("SELECT * FROM dss_guide_content WHERE path=#{path} ORDER BY type,seq") + List getGuideContentListByPath(@Param("path") String path); + + @Select("SELECT content FROM dss_guide_content WHERE id=#{id}") + String getGuideContentById(@Param("id") long id); + + @Update("UPDATE dss_guide_content SET content =#{content},content_html =#{contentHtml} WHERE id =#{id}") + void updateGuideContentById(@Param("id") long id, @Param("content") String content, @Param("contentHtml") String contentHtml); + + int batchInsert(@Param("list") List list); + + void initContentId(); + +// @Update("UPDATE dss_guide_content SET `is_delete` = 1,`update_time` = NOW() WHERE `id` = #{id}") +// void deleteGuideContent(@Param("id") Long id); +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideGroupMapper.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideGroupMapper.java new file mode 100644 index 000000000..81c9677bb --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/GuideGroupMapper.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.guide.server.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.guide.server.entity.GuideGroup; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +@Mapper +public interface GuideGroupMapper extends BaseMapper { + GuideGroup getGuideGroupByPath(@Param("path") String path); + + List getAllGuideGroupDetails(); + + int batchInsert(@Param("list") List list); + +// @Update("UPDATE dss_guide_group SET `is_delete` = 1,`update_time` = NOW() WHERE `id` = #{id}") +// void deleteGuideGroup(@Param("id") Long id); +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideCatalogMapper.xml b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideCatalogMapper.xml new file mode 100644 index 000000000..c5c0b7f64 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideCatalogMapper.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + id, parent_id, title, description, create_by, create_time, update_by, update_time + + + \ No newline at end of file diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideChapterMapper.xml b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideChapterMapper.xml new file mode 100644 index 000000000..5b6c3e7d6 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideChapterMapper.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + id, catalog_id, title, title_alias, content, content_html, create_by, create_time, + update_by, update_time + + + insert into dss_guide_chapter + (catalog_id, title, title_alias, content, content_html, create_by, create_time, update_by, + update_time) + values + + (#{item.catalogId,jdbcType=BIGINT}, #{item.title,jdbcType=VARCHAR}, #{item.titleAlias,jdbcType=VARCHAR}, + #{item.content,jdbcType=LONGVARCHAR}, #{item.contentHtml,jdbcType=LONGVARCHAR}, + #{item.createBy,jdbcType=VARCHAR}, #{item.createTime,jdbcType=TIMESTAMP}, #{item.updateBy,jdbcType=VARCHAR}, + #{item.updateTime,jdbcType=TIMESTAMP}) + + + \ No newline at end of file diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideContentMapper.xml b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideContentMapper.xml new file mode 100644 index 000000000..e1f702515 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideContentMapper.xml @@ -0,0 +1,22 @@ + + + + + + insert into dss_guide_content + (group_id, `path`, title, title_alias, seq, `type`, content, content_html, create_by, + create_time, update_by, update_time, is_delete) + values + + (#{item.groupId}, #{item.path}, #{item.title}, + #{item.titleAlias}, #{item.seq}, #{item.type}, + #{item.content}, #{item.contentHtml}, + #{item.createBy}, #{item.createTime}, #{item.updateBy}, + #{item.updateTime}, #{item.isDelete}) + + + + + \ No newline at end of file diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideGroupMapper.xml b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideGroupMapper.xml new file mode 100644 index 000000000..7f44c14fa --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/dao/impl/GuideGroupMapper.xml @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + insert into dss_guide_group + (`id`,`path`, title, description, create_by, create_time, update_by, update_time, is_delete) + values + + (#{item.id},#{item.path}, #{item.title}, #{item.description}, + #{item.createBy}, #{item.createTime}, #{item.updateBy}, + #{item.updateTime}, #{item.isDelete}) + + + \ No newline at end of file diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideCatalog.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideCatalog.java new file mode 100644 index 000000000..f5e299d0d --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideCatalog.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.guide.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; + +@Data +@TableName(value = "dss_guide_catalog") +public class GuideCatalog { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + private Long parentId; + + private String title; + private String description; + + private String createBy; + private String updateBy; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date updateTime; + //private Integer isDelete; +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideChapter.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideChapter.java new file mode 100644 index 000000000..25091959d --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideChapter.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.guide.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; + +@Data +@TableName(value = "dss_guide_chapter") +public class GuideChapter { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + private Long catalogId; + + private String title; + private String titleAlias; + private String content; + private String contentHtml; + + private String createBy; + private String updateBy; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date updateTime; + //private Integer isDelete; +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideContent.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideContent.java new file mode 100644 index 000000000..43b37c061 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideContent.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.dss.guide.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; + +@Data +@TableName(value = "dss_guide_content") +public class GuideContent { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + private Long groupId; + private String path; + + private Integer type; + private String title; + private String titleAlias; + private String seq; + private String content; + private String contentHtml; + + private String createBy; + private String updateBy; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date updateTime; + private Integer isDelete; +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideGroup.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideGroup.java new file mode 100644 index 000000000..187279e55 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/GuideGroup.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.dss.guide.server.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; +import org.springframework.format.annotation.DateTimeFormat; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +@Data +@TableName(value = "dss_guide_group") +public class GuideGroup implements Serializable { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + private String path; + private String title; + private String description; + + private String createBy; + private String updateBy; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date updateTime; + private Integer isDelete; + + private List children; + +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/response/GuideCatalogDetail.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/response/GuideCatalogDetail.java new file mode 100644 index 000000000..f20f4fe05 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/entity/response/GuideCatalogDetail.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.guide.server.entity.response; + +import com.webank.wedatasphere.dss.guide.server.entity.GuideCatalog; +import com.webank.wedatasphere.dss.guide.server.entity.GuideChapter; +import lombok.Data; + +import java.util.List; + +@Data +public class GuideCatalogDetail { + private Long id; + + private List childrenCatalog; + private List childrenChapter; +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/GuideQueryRestful.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/GuideQueryRestful.java new file mode 100644 index 000000000..991533b6f --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/GuideQueryRestful.java @@ -0,0 +1,65 @@ +package com.webank.wedatasphere.dss.guide.server.restful; + +import com.webank.wedatasphere.dss.guide.server.entity.GuideChapter; +import com.webank.wedatasphere.dss.guide.server.service.GuideCatalogService; +import com.webank.wedatasphere.dss.guide.server.service.GuideChapterService; +import com.webank.wedatasphere.dss.guide.server.service.GuideGroupService; +import lombok.AllArgsConstructor; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; + +@RestController +@RequestMapping(path = "/dss/guide/query", produces = {"application/json"}) +@AllArgsConstructor +public class GuideQueryRestful { + private static final Logger logger = LoggerFactory.getLogger(GuideQueryRestful.class); + + private GuideGroupService guideGroupService; + private GuideCatalogService guideCatalogService; + private GuideChapterService guideChapterService; + + @RequestMapping(path = "/groupdetail", method = RequestMethod.GET) + public Message queryGudieContent(@RequestParam String path) { + return Message.ok().data("result", guideGroupService.queryGuideGroupByPath(path)); + } + + @RequestMapping(path = "/guidecatalog/top", method = RequestMethod.GET) + public Message queryGuideCatalogListForTop() { + return Message.ok().data("result", guideCatalogService.queryGuideCatalogListForTop()); + } + + @RequestMapping(path = "/guidecatalog/{id}/detail", method = RequestMethod.GET) + public Message queryGuideCatalogDetailById(@PathVariable Long id) { + return Message.ok().data("result", guideCatalogService.queryGuideCatalogDetailById(id)); + } + + @RequestMapping(path = "/guidechapter", method = RequestMethod.GET) + public Message queryGuideChapter(@RequestParam("keyword") String keyword, + @RequestParam("pageNow") Integer pageNow, @RequestParam("pageSize") Integer pageSize) { + if (pageNow == null) { + pageNow = 1; + } + if (pageSize == null) { + pageSize = 20; + } + + List totals = new ArrayList<>(); + List guideChapterList = guideChapterService.searchGuideChapterList(keyword, totals, pageNow, pageSize); + return Message.ok().data("result", guideChapterList).data("total", totals.get(0)); + } + + @RequestMapping(path = "/guidechapter/{id}", method = RequestMethod.GET) + public Message queryGuideChapter(@PathVariable Long id) { + return Message.ok().data("result", guideChapterService.queryGuideChapterById(id)); + } + +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/KnowledgeGuideAdminRestful.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/KnowledgeGuideAdminRestful.java new file mode 100644 index 000000000..dc956d498 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/KnowledgeGuideAdminRestful.java @@ -0,0 +1,240 @@ +package com.webank.wedatasphere.dss.guide.server.restful; + +import com.webank.wedatasphere.dss.guide.server.conf.GuideConf; +import com.webank.wedatasphere.dss.guide.server.entity.GuideCatalog; +import com.webank.wedatasphere.dss.guide.server.entity.GuideChapter; +import com.webank.wedatasphere.dss.guide.server.service.GuideCatalogService; +import com.webank.wedatasphere.dss.guide.server.service.GuideChapterService; +import com.webank.wedatasphere.dss.guide.server.service.GuideGroupService; +import com.webank.wedatasphere.dss.guide.server.util.FileUtils; +import com.webank.wedatasphere.dss.guide.server.util.ShellUtils; +import lombok.AllArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.PostConstruct; +import javax.servlet.http.HttpServletRequest; +import java.io.File; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@RestController +@RequestMapping(path = "/dss/guide/admin", produces = {"application/json"}) +@AllArgsConstructor +public class KnowledgeGuideAdminRestful { + private static final Logger logger = LoggerFactory.getLogger(KnowledgeGuideAdminRestful.class); + + private GuideCatalogService guideCatalogService; + private GuideChapterService guideChapterService; + + private GuideGroupService guideGroupService; + + private final static String SUMMARY = "SUMMARY.md"; + + private final static String SHELL_COMMAND_HOST_IP = "hostname -i"; + + private final static String MODEL_GITBOOK_SYNC = "gitbook"; + + + /** + * 知识库目录接口 + */ + @RequestMapping(path = "/guidecatalog", method = RequestMethod.POST) + public Message saveGuideCatalog(HttpServletRequest request, @RequestBody GuideCatalog guideCatalog) { + String userName = SecurityFilter.getLoginUsername(request); + if (null == guideCatalog.getId()) { + guideCatalog.setCreateBy(userName); + guideCatalog.setCreateTime(new Date(System.currentTimeMillis())); + } else { + guideCatalog.setUpdateBy(userName); + guideCatalog.setUpdateTime(new Date(System.currentTimeMillis())); + } + + boolean flag = guideCatalogService.saveGuideCatalog(guideCatalog); + if (flag) { + return Message.ok("保存成功"); + } else { + return Message.error("保存失败"); + } + } + + @RequestMapping(path = "/guidecatalog/{id}/delete", method = RequestMethod.POST) + public Message deleteGroup(@PathVariable Long id) { + guideCatalogService.deleteGuideCatalog(id); + Message message = Message.ok("删除成功"); + return message; + } + + @RequestMapping(path = "/guidecatalog/top", method = RequestMethod.GET) + public Message queryGuideCatalogListForTop() { + return Message.ok().data("result", guideCatalogService.queryGuideCatalogListForTop()); + } + + @RequestMapping(path = "/guidecatalog/{id}/detail", method = RequestMethod.GET) + public Message queryGuideCatalogDetailById(@PathVariable Long id) { + return Message.ok().data("result", guideCatalogService.queryGuideCatalogDetailById(id)); + } + + + /** + * 知识库文档接口 + */ + @RequestMapping(path = "/guidechapter", method = RequestMethod.POST) + public Message saveGuideChapter(HttpServletRequest request, @RequestBody GuideChapter guideChapter) { + String userName = SecurityFilter.getLoginUsername(request); + if (null == guideChapter.getId()) { + guideChapter.setCreateBy(userName); + guideChapter.setCreateTime(new Date(System.currentTimeMillis())); + } else { + guideChapter.setUpdateBy(userName); + guideChapter.setUpdateTime(new Date(System.currentTimeMillis())); + } + + boolean flag = guideChapterService.saveGuideChapter(guideChapter); + if (flag) { + return Message.ok("保存成功"); + } else { + return Message.error("保存失败"); + } + } + + @RequestMapping(path = "/guidechapter/{id}/delete", method = RequestMethod.POST) + public Message deleteGuideChapter(@PathVariable Long id) { + guideChapterService.deleteGuideChapter(id); + Message message = Message.ok("删除成功"); + return message; + } + + @RequestMapping(path = "/guidechapter/{id}", method = RequestMethod.GET) + public Message queryGuideChapter(@PathVariable Long id) { + return Message.ok().data("result", guideChapterService.queryGuideChapterById(id)); + } + + @RequestMapping(path = "/guidechapter/uploadImages", method = RequestMethod.POST) + public Message multFileUpload(@RequestParam(required = true) List files) { + if (null == files || files.size() == 0) { + return Message.error("没有上传文件"); + } + + List> totalResult = new ArrayList>(); + // 要上传的目标文件存放的绝对路径 + final String localPath = GuideConf.GUIDE_CHAPTER_IMAGES_PATH.getValue(); + for (MultipartFile file : files) { + Map result = new HashMap(); + String result_msg = ""; + + if (file.getSize() > 5 * 1024 * 1024) { + result_msg = "图片大小不能超过5M"; + } else { + //判断上传文件格式 + String fileType = file.getContentType(); + if (fileType.equals("image/jpeg") || fileType.equals("image/png") || fileType.equals("image/jpg")) { + //获取文件名 + String fileName = file.getOriginalFilename(); + //获取文件后缀名 + String suffixName = fileName.substring(fileName.lastIndexOf(".")); + //重新生成文件名 + fileName = "knowledge-" + UUID.randomUUID() + suffixName; + if (FileUtils.upload(file, localPath, fileName)) { + String relativePath = fileName; + result.put("relativePath", relativePath); + result_msg = "图片上传成功"; + } else { + result_msg = "图片上传失败"; + } + } else { + result_msg = "图片格式不正确"; + } + } + result.put("result_msg", result_msg); + totalResult.add(result); + } + return Message.ok().data("result", totalResult); + } + + @RequestMapping(path = "/guidechapter/uploadImage", method = RequestMethod.POST) + public Message fileUpload(@RequestParam(required = true) MultipartFile file) { + if (null == file) { + return Message.error("没有上传文件"); + } + + final String imagesPath = GuideConf.GUIDE_CHAPTER_IMAGES_PATH.getValue(); + + if (file.getSize() > 5 * 1024 * 1024) { + return Message.error("图片大小不能超过5M"); + } else { + //判断上传文件格式 + String fileType = file.getContentType(); + if (fileType.equals("image/jpeg") || fileType.equals("image/png") || fileType.equals("image/jpg")) { + //获取文件名 + String fileName = file.getOriginalFilename(); + //获取文件后缀名 + String suffixName = fileName.substring(fileName.lastIndexOf(".")); + //重新生成文件名 + fileName = UUID.randomUUID() + suffixName; + if (FileUtils.upload(file, imagesPath, fileName)) { + return Message.ok().data("result", fileName); + } else { + return Message.error("图片上传失败"); + } + } else { + return Message.error("图片格式不正确"); + } + } + + } + + @PostConstruct + public void syncKnowledge() { + final String summaryPath = GuideConf.HOST_GITBOOK_PATH.getValue() + File.separator + SUMMARY; + final String savePath = GuideConf.TARGET_GITBOOK_PATH.getValue() + File.separator + "gitbook_books"; + final String scpCommand = "scp -r " + + " hadoop@" + GuideConf.HOST_IP_ADDRESS.getValue() + ":" + + GuideConf.HOST_GITBOOK_PATH.getValue() + " " + + GuideConf.TARGET_GITBOOK_PATH.getValue(); + String delMkdir = "rm -rf " + savePath; + logger.info("开始执行定时任务..."); + Utils.defaultScheduler().scheduleAtFixedRate(() -> { + try { + if (StringUtils.equals(GuideConf.GUIDE_SYNC_MODEL.getValue(), MODEL_GITBOOK_SYNC)) { + String hostIp = ShellUtils.callShellQuery(SHELL_COMMAND_HOST_IP); + //如果不是当前节点,则需要拷贝文件 + if (!StringUtils.equals(hostIp, GuideConf.HOST_IP_ADDRESS.getValue())) { + //判断文件是否存在 + boolean flag = FileUtils.fileExist(savePath); + if (flag) { + //删除文件 + ShellUtils.callShellByExec(delMkdir); + } + //拷贝文件到相应节点 + ShellUtils.callShellByExec(scpCommand); + } + }else { + guideCatalogService.syncKnowledge(summaryPath, GuideConf.SUMMARY_IGNORE_MODEL.getValue()); + guideGroupService.asyncGuide(summaryPath, GuideConf.SUMMARY_IGNORE_MODEL.getValue()); + } + } catch (Exception e) { + logger.error("定时任务执行异常:" + e); + throw new RuntimeException(e); + } + logger.info("定时任务执行结束!!!"); + }, 0, 2, TimeUnit.HOURS); + } + +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/PageGuideAdminRestful.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/PageGuideAdminRestful.java new file mode 100644 index 000000000..4850ebafa --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/restful/PageGuideAdminRestful.java @@ -0,0 +1,190 @@ +package com.webank.wedatasphere.dss.guide.server.restful; + +import com.webank.wedatasphere.dss.guide.server.conf.GuideConf; +import com.webank.wedatasphere.dss.guide.server.entity.GuideContent; +import com.webank.wedatasphere.dss.guide.server.entity.GuideGroup; +import com.webank.wedatasphere.dss.guide.server.service.GuideContentService; +import com.webank.wedatasphere.dss.guide.server.service.GuideGroupService; +import com.webank.wedatasphere.dss.guide.server.util.FileUtils; +import lombok.AllArgsConstructor; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@RestController +@RequestMapping(path = "/dss/guide/admin", produces = {"application/json"}) +@AllArgsConstructor +public class PageGuideAdminRestful { + private static final Logger logger = LoggerFactory.getLogger(PageGuideAdminRestful.class); + + private GuideGroupService guideGroupService; + private GuideContentService guideContentService; + + @RequestMapping(path = "/guidegroup", method = RequestMethod.POST) + public Message saveGuideGroup(HttpServletRequest request, @RequestBody GuideGroup guideGroup) { + String userName = SecurityFilter.getLoginUsername(request); + if (null == guideGroup.getId()) { + guideGroup.setCreateBy(userName); + guideGroup.setCreateTime(new Date(System.currentTimeMillis())); + } else { + guideGroup.setUpdateBy(userName); + guideGroup.setUpdateTime(new Date(System.currentTimeMillis())); + } + + boolean flag = guideGroupService.saveGuideGroup(guideGroup); + if (flag) { + return Message.ok("保存成功"); + } else { + return Message.error("保存失败"); + } + } + + @RequestMapping(path = "/guidegroup", method = RequestMethod.GET) + public Message queryGuideGroup() { + return Message.ok().data("result", guideGroupService.getAllGuideGroupDetails()); + } + + @RequestMapping(path = "/guidegroup/{id}/delete", method = RequestMethod.POST) + public Message deleteGuideGroup(@PathVariable Long id) { + guideGroupService.deleteGuideGroup(id); + Message message = Message.ok("删除成功"); + return message; + } + + @RequestMapping(path = "/guidecontent", method = RequestMethod.POST) + public Message saveGuideContent(HttpServletRequest request, @RequestBody GuideContent guideConent) { + String userName = SecurityFilter.getLoginUsername(request); + if (null == guideConent.getId()) { + guideConent.setCreateBy(userName); + guideConent.setCreateTime(new Date(System.currentTimeMillis())); + } else { + guideConent.setUpdateBy(userName); + guideConent.setUpdateTime(new Date(System.currentTimeMillis())); + } + + boolean flag = guideContentService.saveGuideContent(guideConent); + if (flag) { + return Message.ok("保存成功"); + } else { + return Message.error("保存失败"); + } + } + + @RequestMapping(path = "/guidecontent", method = RequestMethod.GET) + public Message queryGuideContent(@RequestParam String path) { + return Message.ok().data("result", guideContentService.queryGuideContentByPath(path)); + } + + @RequestMapping(path = "/guidecontent/{id}", method = RequestMethod.GET) + public Message queryGuideContent(@PathVariable Long id) { + return Message.ok().data("result", guideContentService.getGuideContent(id)); + } + + @RequestMapping(path = "/guidecontent/{id}/content", method = RequestMethod.POST) + public Message updateGuideContent(@PathVariable Long id, @RequestBody Map map) { + try { + guideContentService.updateGuideContentById(id, map); + return Message.ok("更新成功"); + } catch (Exception ex) { + logger.error("ERROR", "Error found: ", ex); + return Message.error(ex.getMessage()); + } + } + + @RequestMapping(path = "/guidecontent/{id}/delete", method = RequestMethod.POST) + public Message deleteContent(@PathVariable Long id) { + guideContentService.deleteGuideContent(id); + Message message = Message.ok("删除成功"); + return message; + } + + @RequestMapping(path = "/guidecontent/uploadImages", method = RequestMethod.POST) + public Message multFileUpload(@RequestParam(required = true) List files) { + if (null == files || files.size() == 0) { + return Message.error("没有上传文件"); + } + + List> totalResult = new ArrayList>(); + // 要上传的目标文件存放的绝对路径 + final String localPath = GuideConf.GUIDE_CONTENT_IMAGES_PATH.getValue(); + for (MultipartFile file : files) { + Map result = new HashMap(); + String result_msg = ""; + + if (file.getSize() > 5 * 1024 * 1024) { + result_msg = "图片大小不能超过5M"; + } else { + //判断上传文件格式 + String fileType = file.getContentType(); + if (fileType.equals("image/jpeg") || fileType.equals("image/png") || fileType.equals("image/jpg")) { + //获取文件名 + String fileName = file.getOriginalFilename(); + //获取文件后缀名 + String suffixName = fileName.substring(fileName.lastIndexOf(".")); + //重新生成文件名 + fileName = "page-" + UUID.randomUUID() + suffixName; + if (FileUtils.upload(file, localPath, fileName)) { + String relativePath = fileName; + result.put("relativePath", relativePath); + result_msg = "图片上传成功"; + } else { + result_msg = "图片上传失败"; + } + } else { + result_msg = "图片格式不正确"; + } + } + result.put("result_msg", result_msg); + totalResult.add(result); + } + return Message.ok().data("result", totalResult); + } + + @RequestMapping(path = "/guidecontent/uploadImage", method = RequestMethod.POST) + public Message fileUpload(@RequestParam(required = true) MultipartFile file) { + if (null == file) { + return Message.error("没有上传文件"); + } + + final String imagesPath = GuideConf.GUIDE_CONTENT_IMAGES_PATH.getValue(); + + if (file.getSize() > 5 * 1024 * 1024) { + return Message.error("图片大小不能超过5M"); + } else { + //判断上传文件格式 + String fileType = file.getContentType(); + if (fileType.equals("image/jpeg") || fileType.equals("image/png") || fileType.equals("image/jpg")) { + //获取文件名 + String fileName = file.getOriginalFilename(); + //获取文件后缀名 + String suffixName = fileName.substring(fileName.lastIndexOf(".")); + //重新生成文件名 + fileName = UUID.randomUUID() + suffixName; + if (FileUtils.upload(file, imagesPath, fileName)) { + return Message.ok().data("result", fileName); + } else { + return Message.error("图片上传失败"); + } + } else { + return Message.error("图片格式不正确"); + } + } + + } +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideCatalogService.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideCatalogService.java new file mode 100644 index 000000000..ab96ba714 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideCatalogService.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.guide.server.service; + +import com.webank.wedatasphere.dss.guide.server.entity.GuideCatalog; +import com.webank.wedatasphere.dss.guide.server.entity.response.GuideCatalogDetail; + +import java.util.List; + +public interface GuideCatalogService { + public boolean saveGuideCatalog(GuideCatalog guideCatalog); + + public void deleteGuideCatalog(Long id); + + public List queryGuideCatalogListForTop(); + + public GuideCatalogDetail queryGuideCatalogDetailById(Long id); + + public void syncKnowledge(String summaryPath, String ignoreModel) throws Exception; +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideChapterService.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideChapterService.java new file mode 100644 index 000000000..bcb7da6c5 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideChapterService.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.guide.server.service; + +import com.webank.wedatasphere.dss.guide.server.entity.GuideChapter; + +import java.util.List; + +public interface GuideChapterService { + public boolean saveGuideChapter(GuideChapter guideChapter); + + public void deleteGuideChapter(Long id); + + public GuideChapter queryGuideChapterById(Long id); + + public List searchGuideChapterList(String keyword, List totals, Integer pageNow, Integer pageSize); +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideContentService.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideContentService.java new file mode 100644 index 000000000..5775fbc4b --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideContentService.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.guide.server.service; + +import com.webank.wedatasphere.dss.guide.server.entity.GuideContent; +import com.webank.wedatasphere.dss.guide.server.util.GuideException; + +import java.util.List; +import java.util.Map; + +public interface GuideContentService { + public boolean saveGuideContent(GuideContent guideGroup); + + public GuideContent getGuideContent(long id); + + public List queryGuideContentByPath(String path); + + public void updateGuideContentById(long id, Map map) throws GuideException; + + public void deleteGuideContent(Long id); +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideGroupService.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideGroupService.java new file mode 100644 index 000000000..9b92bcc06 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/GuideGroupService.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.guide.server.service; + +import com.webank.wedatasphere.dss.guide.server.entity.GuideGroup; +import com.webank.wedatasphere.dss.guide.server.util.GuideException; + +import java.util.List; + +public interface GuideGroupService { + public boolean saveGuideGroup(GuideGroup guideGroup); + + public GuideGroup queryGuideGroupByPath(String path); + + public List getAllGuideGroupDetails(); + + public void deleteGuideGroup(Long id); + + public void asyncGuide(String summaryPath, String ignoreModel) throws GuideException; +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideCatalogServiceImpl.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideCatalogServiceImpl.java new file mode 100644 index 000000000..f1115ffdc --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideCatalogServiceImpl.java @@ -0,0 +1,184 @@ +package com.webank.wedatasphere.dss.guide.server.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.guide.server.conf.GuideConf; +import com.webank.wedatasphere.dss.guide.server.dao.GuideCatalogMapper; +import com.webank.wedatasphere.dss.guide.server.dao.GuideChapterMapper; +import com.webank.wedatasphere.dss.guide.server.entity.GuideCatalog; +import com.webank.wedatasphere.dss.guide.server.entity.GuideChapter; +import com.webank.wedatasphere.dss.guide.server.entity.response.GuideCatalogDetail; +import com.webank.wedatasphere.dss.guide.server.service.GuideCatalogService; +import com.webank.wedatasphere.dss.guide.server.util.GuideException; +import com.webank.wedatasphere.dss.guide.server.util.MdAnalysis; +import lombok.AllArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Service +@AllArgsConstructor +public class GuideCatalogServiceImpl extends ServiceImpl implements GuideCatalogService { + + private static final Logger logger = LoggerFactory.getLogger(GuideCatalogServiceImpl.class); + private GuideCatalogMapper guideCatalogMapper; + private GuideChapterMapper guideChapterMapper; + + @Override + public boolean saveGuideCatalog(GuideCatalog guideCatalog) { + Long id = guideCatalog.getId(); + + if(id != null){ + return this.updateById(guideCatalog); + } + else { + return this.save(guideCatalog); + } + } + + @Override + public void deleteGuideCatalog(Long id) { + this.removeById(id); + } + + @Override + public List queryGuideCatalogListForTop() { + return guideCatalogMapper.queryGuideCatalogListForTop(); + } + + @Override + public GuideCatalogDetail queryGuideCatalogDetailById(Long id) { + GuideCatalogDetail guideCatalogDetail =new GuideCatalogDetail(); + + guideCatalogDetail.setId(id); + guideCatalogDetail.setChildrenCatalog(guideCatalogMapper.queryGuideCatalogChildrenById(id)); + guideCatalogDetail.setChildrenChapter(guideChapterMapper.queryGuideChapterListByCatalogId(id)); + + return guideCatalogDetail; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void syncKnowledge(String summaryPath, String ignoreModel) throws Exception { + logger.info("====================初始化=================="); + chapterInit(); + logger.info("summary文件路径:" + summaryPath); + //父级目录 + String parentPath = new File(summaryPath).getParent(); + logger.info("父级路径:" + parentPath); + List catalogs = null; + List chapters = null; + try { + //1.解析SUMMARY.md文件 + List>> maps = MdAnalysis.analysisMd(summaryPath,"knowledge", ignoreModel); + //2.解析maps并插入dss_guide_catalog表 + catalogs = new ArrayList<>(); + chapters = new ArrayList<>(); + for (Map> map : maps) { + for (Map.Entry tag : map.entrySet()) { + //可以转为int类型则为一级标题 + if (MdAnalysis.isStr2Num(String.valueOf(tag.getKey()))) { + //参数组装 + catalogs.add(buildParams(map, tag, Long.valueOf(tag.getKey().toString()), -1l)); + } + + //解析二级标题 + if (MdAnalysis.isSecondOrThird(String.valueOf(tag.getKey())) == 1) { + //[1-1, 1-2, 1-3, 1-4, 1-5, 1-6, 1-7, 1-8, 1-9, 1-10, 2-1, 2-2, 3-1, 3-2, 3-3] + String secondTag = String.valueOf(tag.getKey()); + //将split[1]作为id,split[0]作为parentId + String[] split = secondTag.split("-"); + //参数组装 + if (null != map.get(tag.getKey()).get("file")) { + String mdFilePath = parentPath + map.get(tag.getKey()).get("file"); + chapters.add(buildChapterParams(map, tag, Long.valueOf(split[0]), mdFilePath)); + } else { + catalogs.add(buildParams(map, tag, Long.valueOf(split[1]) + MdAnalysis.getRootCount(), Long.valueOf(split[0]))); + } + } + + //解析三级标题 + if (MdAnalysis.isSecondOrThird(String.valueOf(tag.getKey())) == 2) { + //[1-1-1, 1-1-2, 1-1-3, 1-1-4, 1-2-1, 1-2-2, 1-2-3, 1-2-4, 1-2-5, 1-2-6, 1-2-7, 1-2-8, 1-2-9, 1-2-10, 1-2-11, 1-2-12, 1-2-13, 1-2-14, 1-2-15, 1-2-16, 1-2-17, 1-2-18, 1-2-19, 1-2-20, 1-2-21, 1-2-22, 1-2-23, 1-2-24, 1-2-25, 1-2-26, 1-3-1, 1-4-1, 1-4-2, 1-4-3, 1-4-4, 1-4-5, 1-4-6, 1-4-7, 1-4-8, 1-4-9, 1-4-10, 1-4-11, 1-4-12, 1-4-13, 1-4-14, 1-4-15, 1-4-16, 1-4-17, 1-4-18, 1-5-1, 1-5-2, 1-5-3, 1-5-4, 1-5-5, 1-5-6, 1-5-7, 1-5-8, 1-5-9, 1-5-10, 1-5-11, 1-5-12, 1-5-13, 1-5-14, 1-5-15, 1-5-16, 1-6-1, 1-6-2, 1-6-3, 1-6-4, 1-6-5, 1-6-6, 1-7-1, 1-7-2, 1-7-3, 1-7-4, 1-7-5, 1-7-6, 1-7-7, 1-7-8, 1-7-9, 1-7-10, 1-7-11, 1-7-12, 1-7-13, 1-7-14, 1-8-1, 1-8-2, 1-8-3, 1-8-4, 1-8-5, 1-9-1, 1-9-2, 1-9-3] + String secondTag = String.valueOf(tag.getKey()); + //split[1]作为catalogId + String[] split = secondTag.split("-"); + //获取summary.md文件中的md文件 + String mdFilePath = parentPath + map.get(tag.getKey()).get("file"); + //组装参数存入dss_guide_chapter表 + chapters.add(buildChapterParams(map, tag, Long.valueOf(split[1]) + MdAnalysis.getRootCount(), mdFilePath)); + } + } + } + } catch (Exception e) { + logger.error("服务异常:"+ e); + throw new RuntimeException(e); + } + + logger.info("catalogs=======>>>>"+ catalogs); + if (!CollectionUtils.isEmpty(catalogs)) { + //批量插入到dss_guide_catalog表 + logger.info("=========开始批量插入dss_guide_catalog表============"); + int flag = guideCatalogMapper.batchInsert(catalogs); + if(flag == 0){ + logger.info("=========批量插入失败dss_guide_catalog表============"); + throw new GuideException("批量插入失败!"); + } + } + logger.info("chapters=======>>>>"+ chapters); + if (!CollectionUtils.isEmpty(chapters)) { + //批量插入到dss_guide_chapter表 + logger.info("=========开始批量插入dss_guide_chapter表============"); + int flag = guideChapterMapper.batchInsert(chapters); + if(flag == 0){ + logger.info("=========批量插入失败dss_guide_chapter表============"); + throw new GuideException("批量插入失败!"); + } + } + logger.info("知识库同步完成。。。"); + } + + private GuideCatalog buildParams(Map> map, Map.Entry tag, Long id, Long parentId) { + GuideCatalog guideCatalog = new GuideCatalog(); + guideCatalog.setId(id); + guideCatalog.setParentId(parentId); + guideCatalog.setTitle(map.get(tag.getKey()).get("title")); + guideCatalog.setCreateTime(new Date()); + guideCatalog.setUpdateTime(new Date()); + return guideCatalog; + } + + private GuideChapter buildChapterParams(Map> map, Map.Entry tag, Long parentId, String mdFilePath) throws IOException { + GuideChapter guideChapter = new GuideChapter(); + guideChapter.setCatalogId(parentId); + guideChapter.setTitle(map.get(tag.getKey()).get("title")); + if (!StringUtils.isEmpty(mdFilePath)) { + logger.info("GuideChapter文件路径:"+ mdFilePath); + String content = MdAnalysis.readMd(mdFilePath); + guideChapter.setContent(content); + guideChapter.setContentHtml(MdAnalysis.changeHtmlTagA(MdAnalysis.markdown2Html(mdFilePath))); + } + guideChapter.setUpdateTime(new Date()); + guideChapter.setCreateTime(new Date()); + return guideChapter; + } + + private void chapterInit(){ + //初始化dss_guide_catalog表 + guideCatalogMapper.delete(null); + //初始化dss_guide_chapter表 + guideChapterMapper.delete(null); + //初始化dss_guide_chapter表id + guideChapterMapper.initChapterId(); + MdAnalysis.rootCountInit(); + } +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideChapterServiceImpl.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideChapterServiceImpl.java new file mode 100644 index 000000000..2bfc88ae0 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideChapterServiceImpl.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.guide.server.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.guide.server.dao.GuideChapterMapper; +import com.webank.wedatasphere.dss.guide.server.entity.GuideChapter; +import com.webank.wedatasphere.dss.guide.server.service.GuideChapterService; +import lombok.AllArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@AllArgsConstructor +public class GuideChapterServiceImpl extends ServiceImpl implements GuideChapterService { + private GuideChapterMapper guideChapterMapper; + + @Override + public boolean saveGuideChapter(GuideChapter guideChapter) { + Long id = guideChapter.getId(); + + if(id != null){ + return this.updateById(guideChapter); + } + else { + return this.save(guideChapter); + } + } + + @Override + public void deleteGuideChapter(Long id) { + this.removeById(id); + } + + @Override + public GuideChapter queryGuideChapterById(Long id) { + return this.getById(id); + } + + @Override + public List searchGuideChapterList(String keyword, List totals, Integer pageNow, Integer pageSize) { + PageHelper.startPage(pageNow, pageSize, true); + // MYSQL LIKE % _: LIKE '%\_%', LIKE '%\%%' + if(keyword !=null) { + if ("_".equalsIgnoreCase(keyword.trim())) { + keyword = "\\_"; + } + if ("%".equalsIgnoreCase(keyword.trim())) { + keyword = "\\%"; + } + } + List guideChapterList = guideChapterMapper.searchGuideChapterListByKeyword(keyword); + PageInfo pageInfo = new PageInfo<>(guideChapterList); + totals.add(pageInfo.getTotal()); + + return guideChapterList; + } +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideContentServiceImpl.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideContentServiceImpl.java new file mode 100644 index 000000000..bf52c23d7 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideContentServiceImpl.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.guide.server.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.guide.server.dao.GuideContentMapper; +import com.webank.wedatasphere.dss.guide.server.entity.GuideContent; +import com.webank.wedatasphere.dss.guide.server.service.GuideContentService; +import com.webank.wedatasphere.dss.guide.server.util.GuideException; +import lombok.AllArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; + +@Service +@AllArgsConstructor +public class GuideContentServiceImpl extends ServiceImpl implements GuideContentService { + private GuideContentMapper guideContentMapper; + + @Override + public boolean saveGuideContent(GuideContent guideContent) { + Long id = guideContent.getId(); + + if(id != null){ + return this.updateById(guideContent); + } + else { + return this.save(guideContent); + } + } + + @Override + public GuideContent getGuideContent(long id){ + return guideContentMapper.selectById(id); + } + + @Override + public List queryGuideContentByPath(String path) { + return guideContentMapper.getGuideContentListByPath(path); + } + + + @Override + public void updateGuideContentById(long id, Map map) throws GuideException { + Object content = map.get("content"); + if(content == null){ + throw new GuideException("请设置content参数"); + } + Object contentHtml = map.get("contentHtml"); + if(contentHtml == null){ + throw new GuideException("请设置contentHtml参数"); + } + guideContentMapper.updateGuideContentById(id, content.toString(), contentHtml.toString()); + } + + @Override + public void deleteGuideContent(Long id) { + this.removeById(id); + } +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideGroupServiceImpl.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideGroupServiceImpl.java new file mode 100644 index 000000000..18d406509 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/service/impl/GuideGroupServiceImpl.java @@ -0,0 +1,163 @@ +package com.webank.wedatasphere.dss.guide.server.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.guide.server.dao.GuideContentMapper; +import com.webank.wedatasphere.dss.guide.server.dao.GuideGroupMapper; +import com.webank.wedatasphere.dss.guide.server.entity.GuideContent; +import com.webank.wedatasphere.dss.guide.server.entity.GuideGroup; +import com.webank.wedatasphere.dss.guide.server.service.GuideGroupService; +import com.webank.wedatasphere.dss.guide.server.util.GuideException; +import com.webank.wedatasphere.dss.guide.server.util.MdAnalysis; +import lombok.AllArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Service +@AllArgsConstructor +public class GuideGroupServiceImpl extends ServiceImpl implements GuideGroupService { + + private static final Logger logger = LoggerFactory.getLogger(GuideGroupServiceImpl.class); + private GuideGroupMapper guideGroupMapper; + + private GuideContentMapper guideContentMapper; + + @Override + public boolean saveGuideGroup(GuideGroup guideGroup) { + Long id = guideGroup.getId(); + + if(id != null){ + return this.updateById(guideGroup); + } + else { + return this.save(guideGroup); + } + } + + @Override + public GuideGroup queryGuideGroupByPath(String path) { + return guideGroupMapper.getGuideGroupByPath(path); + } + + @Override + public List getAllGuideGroupDetails() { + return guideGroupMapper.getAllGuideGroupDetails(); + } + + @Override + public void deleteGuideGroup(Long id){ + this.removeById(id); + } + + @Override + public void asyncGuide(String summaryPath,String ignoreModel) throws GuideException { + logger.info("====================初始化guide-group guide-content=================="); + guideInit(); + //父级目录 + String parentPath = new File(summaryPath).getParent(); + logger.info("父级路径:" + parentPath); + List guideGroups = new ArrayList<>(); + List guideContents = new ArrayList<>(); + String index = null; + try { + //1.解析SUMMARY.md文件 + List>> maps = MdAnalysis.analysisMd(summaryPath,"guide",ignoreModel); + for(Map> map : maps){ + for (Map.Entry tag : map.entrySet()) { + if(!StringUtils.isEmpty(map.get("url"))){ + index = map.get("url").get("path").trim(); + } + if(MdAnalysis.isStr2Num(String.valueOf(tag.getKey()))){ + guideGroups.add(buildGroupParams(map,tag,Long.valueOf(tag.getKey().toString()),index)); + } + //解析三级标题 + if (MdAnalysis.isSecondOrThird(String.valueOf(tag.getKey())) == 2) { + //[1-1-1, 1-1-2, 1-1-3, 1-1-4, 1-2-1, 1-2-2, 1-2-3, 1-2-4, 1-2-5, 1-2-6, 1-2-7, 1-2-8, 1-2-9, 1-2-10, 1-2-11, 1-2-12, 1-2-13, 1-2-14, 1-2-15, 1-2-16, 1-2-17, 1-2-18, 1-2-19, 1-2-20, 1-2-21, 1-2-22, 1-2-23, 1-2-24, 1-2-25, 1-2-26, 1-3-1, 1-4-1, 1-4-2, 1-4-3, 1-4-4, 1-4-5, 1-4-6, 1-4-7, 1-4-8, 1-4-9, 1-4-10, 1-4-11, 1-4-12, 1-4-13, 1-4-14, 1-4-15, 1-4-16, 1-4-17, 1-4-18, 1-5-1, 1-5-2, 1-5-3, 1-5-4, 1-5-5, 1-5-6, 1-5-7, 1-5-8, 1-5-9, 1-5-10, 1-5-11, 1-5-12, 1-5-13, 1-5-14, 1-5-15, 1-5-16, 1-6-1, 1-6-2, 1-6-3, 1-6-4, 1-6-5, 1-6-6, 1-7-1, 1-7-2, 1-7-3, 1-7-4, 1-7-5, 1-7-6, 1-7-7, 1-7-8, 1-7-9, 1-7-10, 1-7-11, 1-7-12, 1-7-13, 1-7-14, 1-8-1, 1-8-2, 1-8-3, 1-8-4, 1-8-5, 1-9-1, 1-9-2, 1-9-3] + String secondTag = String.valueOf(tag.getKey()); + //split[1]作为catalogId + String[] split = secondTag.split("-"); + //获取summary.md文件中的md文件 + String mdFilePath = parentPath + map.get(tag.getKey()).get("file"); + //组装参数存入dss_guide_chapter表 + guideContents.add(buildContentParams(map, tag, Long.valueOf(split[0]),Integer.valueOf(split[1]), mdFilePath,index)); + } + } + } + } catch (Exception e) { + logger.error("服务异常:"+ e); + throw new RuntimeException(e); + } + + logger.info("guideGroups=======>>>>"+ guideGroups); + if (!CollectionUtils.isEmpty(guideGroups)) { + //批量插入到dss_guide_group表 + logger.info("=========开始批量插入dss_guide_group表============"); + int flag = guideGroupMapper.batchInsert(guideGroups); + if(flag == 0){ + logger.info("=========批量插入失败dss_guide_group表============"); + throw new GuideException("批量插入失败!"); + } + } + logger.info("guideContents=======>>>>"+ guideContents); + if (!CollectionUtils.isEmpty(guideContents)) { + //批量插入到dss_guide_content表 + logger.info("=========开始批量插入dss_guide_content表============"); + int flag = guideContentMapper.batchInsert(guideContents); + if(flag == 0){ + logger.info("=========批量插入失败dss_guide_content表============"); + throw new GuideException("批量插入失败!"); + } + } + logger.info("学习引导同步完成!!!"); + } + + private GuideContent buildContentParams(Map> map, Map.Entry tag, long groupId, int type, String mdFilePath, String index) throws IOException { + GuideContent guideContent = new GuideContent(); + if (!StringUtils.isEmpty(mdFilePath)) { + String content = MdAnalysis.readMd(mdFilePath); + guideContent.setContent(content); + guideContent.setContentHtml(MdAnalysis.changeHtmlTagA(MdAnalysis.markdown2Html(mdFilePath))); + } + guideContent.setType(type); + guideContent.setTitle(map.get(tag.getKey()).get("title")); + guideContent.setGroupId(groupId); + guideContent.setPath(index); + guideContent.setCreateTime(new Date()); + guideContent.setUpdateTime(new Date()); + guideContent.setUpdateBy("system"); + guideContent.setCreateBy("system"); + guideContent.setIsDelete(0); + return guideContent; + } + + + private GuideGroup buildGroupParams(Map> map,Map.Entry tag, Long id,String index){ + GuideGroup guideGroup = new GuideGroup(); + guideGroup.setId(id); + guideGroup.setTitle(map.get(tag.getKey()).get("title")); + guideGroup.setCreateTime(new Date()); + guideGroup.setUpdateTime(new Date()); + guideGroup.setCreateBy("system"); + guideGroup.setUpdateBy("system"); + guideGroup.setPath(index); + guideGroup.setIsDelete(0); + return guideGroup; + } + + private void guideInit(){ + guideGroupMapper.delete(null); + guideContentMapper.delete(null); + //初始化dss_guide_chapter表id + guideContentMapper.initContentId(); + MdAnalysis.rootCountInit(); + } +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/FileUtils.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/FileUtils.java new file mode 100644 index 000000000..4be2d7128 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/FileUtils.java @@ -0,0 +1,48 @@ +package com.webank.wedatasphere.dss.guide.server.util; + +import com.google.common.io.Files; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; + +public class FileUtils { + private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); + + public static boolean upload(MultipartFile file, String path, String fileName) { + String realPath = path + File.separator + fileName; + logger.info("上传文件路径:" + realPath); + + File dest = new File(realPath); + //判断文件父目录是否存在 + if (!dest.getParentFile().exists()) { + dest.getParentFile().mkdir(); + } + try { + byte[] bytes = file.getBytes(); + Files.write(bytes, dest); + return true; + } catch (Exception ex) { + logger.warn("文件上传出错:" + ex.getMessage()); + return false; + } + + } + + /** + * 判断文件是否存在,存在则删除 + * + * @param filePath + * @return + */ + public static boolean fileExist(String filePath) { + logger.info("删除文件路径:" + filePath); + File file = new File(filePath); + if (file.exists()) { + return true; + } + return false; + } +} + diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/GuideException.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/GuideException.java new file mode 100644 index 000000000..3b7cdd85f --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/GuideException.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.dss.guide.server.util; + +public class GuideException extends Exception { + + public GuideException(final String message) { + super(message); + } +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/MdAnalysis.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/MdAnalysis.java new file mode 100644 index 000000000..d60e0b148 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/MdAnalysis.java @@ -0,0 +1,249 @@ +package com.webank.wedatasphere.dss.guide.server.util; + +import com.vladsch.flexmark.html.HtmlRenderer; +import com.vladsch.flexmark.parser.Parser; +import com.vladsch.flexmark.util.ast.Node; +import com.vladsch.flexmark.util.data.MutableDataSet; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class MdAnalysis { + + private static final Logger logger = LoggerFactory.getLogger(MdAnalysis.class); + + private final static Pattern ATTR_PATTERN3 = Pattern.compile("(?<=\\[)(.+?)(?=\\])"); + + private final static Pattern ATTR_PATTERN2 = Pattern.compile("(?<=\\()(.+?)(?=\\))", Pattern.CASE_INSENSITIVE); + + private final static Pattern ATTR_PATTERN1 = Pattern.compile("]*href=(\"([^\"]*)\"|'([^']*)'|([^\\s>]*))[^>]*>"); + + private static int ROOT_COUNT = 0; + + private static int Y = 0; + + private static int Z = 0; + + + /** + * 读取MD文件 + * + * @param filePath + * @return + */ + public static String readMd(String filePath) { + logger.info("开始读取md文件:" + filePath); + File file = new File(filePath); + if (!file.exists()) { + logger.info("文件不存在!" + filePath); + return null; + } + FileReader fr = null; + StringBuffer sb = new StringBuffer(""); + try { + fr = new FileReader(filePath); + BufferedReader br = new BufferedReader(fr); + String line = br.readLine(); + while (line != null) { + sb.append(line); + line = br.readLine(); + } + } catch (IOException e) { + logger.error("文件读取异常===》" + e); + throw new RuntimeException(e); + } finally { + try { + fr.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return sb.toString(); + } + + /** + * 解析SUMMARY.md文件 + * + * @param filePath + * @return + * @throws IOException + */ + public static List>> analysisMd(String filePath, String type, String ignoreModel) throws IOException { + logger.info("开始解析summary.md文件=============》"); + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filePath), "UTF-8")); + List>> mapList = new ArrayList<>(); + String line = null; + boolean flag = false; + while ((line = br.readLine()) != null) { + Map> map = new HashMap<>(); + Map dataMap = new HashMap<>(); + if (StringUtils.equals(type, "guide")) { + if (StringUtils.equals(line.trim(), type)) { + logger.info("开始解析学习引导模块"); + flag = true; + } + if (StringUtils.equals(line.trim(), "knowledge") || ignoreModel.contains(line.trim())) { + flag = false; + } + } + if (StringUtils.equals(type, "knowledge")) { + if (StringUtils.equals(line.trim(), type)) { + logger.info("开始解析知识库模块"); + flag = true; + } + if (StringUtils.equals(line.trim(), "guide") || ignoreModel.contains(line.trim())) { + flag = false; + } + } + if (flag) { + absolveKnowledge(line, dataMap, map, mapList); + } + } + Y = 0; + Z = 0; + return mapList; + } + + private static void absolveKnowledge(String line, Map dataMap, Map> map, List>> mapList) { + if (line.startsWith(" -")) { + ROOT_COUNT++; + Y = 0; + matcherContent(line, dataMap); + map.put(String.valueOf(ROOT_COUNT), dataMap); + mapList.add(map); + } else if (line.startsWith(" -")) { + Y++; + Z = 0; + matcherContent(line, dataMap); + map.put(ROOT_COUNT + "-" + Y, dataMap); + mapList.add(map); + } else if (line.startsWith(" -")) { + Z++; + matcherContent(line, dataMap); + map.put(ROOT_COUNT + "-" + Y + "-" + Z, dataMap); + mapList.add(map); + } else if (line.trim().startsWith("/")) { + dataMap.put("path",line); + map.put("url",dataMap); + mapList.add(map); + } + } + + /** + * 获取跟目录数量 + * + * @return + */ + public static Long getRootCount() { + return Long.valueOf(ROOT_COUNT); + } + + public static void rootCountInit() { + ROOT_COUNT = 0; + } + + ; + + /** + * 解析SUMMARY中的标题和文件 + * + * @param line + * @param dataMap + */ + private static void matcherContent(String line, Map dataMap) { + Matcher matcher1 = ATTR_PATTERN3.matcher(line); + while (matcher1.find()) { + dataMap.put("title", matcher1.group(1) == null ? "" : matcher1.group(1)); + } + Matcher matcher2 = ATTR_PATTERN2.matcher(line); + while (matcher2.find()) { + dataMap.put("file", matcher2.group(1) == null ? "" : matcher2.group(1)); + } + } + + /** + * md转html + * + * @param filePath + * @return + */ + public static String markdown2Html(String filePath) throws IOException { + File file = new File(filePath); + if (!file.exists()) { + return null; + } + MutableDataSet OPTIONS = new MutableDataSet(); + InputStream stream = new FileInputStream(filePath); + String htmlContent = IOUtils.toString(stream, "UTF-8"); + Parser parser = Parser.builder(OPTIONS).build(); + HtmlRenderer renderer = HtmlRenderer.builder(OPTIONS).build(); + Node document = parser.parse(htmlContent); + return renderer.render(document); + } + + public static String changeHtmlTagA(String htmlContent)throws IOException{ + if(StringUtils.isEmpty(htmlContent)){ + return htmlContent; + } + //判断是否含有标签 + if(!htmlContent.contains(""); + return replaceContent; + } + + private static String matcherDir(String line) { + Matcher matcher = ATTR_PATTERN2.matcher(line); + return matcher.find() == true ? matcher.group(1) : null; + } + + /** + * 解析一级标题 + * + * @param str + * @return + */ + public static boolean isStr2Num(String str) { + try { + Integer.parseInt(str); + return true; + } catch (NumberFormatException e) { + return false; + } + } + + /** + * 解析二级三级标题 + * + * @param str + * @return + */ + public static int isSecondOrThird(String str) { + int count = 0; + for (int i = 0; i < str.length(); i++) { + int i1 = str.indexOf("-", i); + if (i1 == i) { + count++; + } + } + return count; + } + +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/ShellUtils.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/ShellUtils.java new file mode 100644 index 000000000..bb497155c --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/ShellUtils.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.guide.server.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +public class ShellUtils { + + private static final Logger logger = LoggerFactory.getLogger(ShellUtils.class); + + /** + * 执行shell命令 + * + * @param shellString + */ + public static void callShellByExec(String shellString) { + try { + logger.error("shellString:" + shellString); + Process process = Runtime.getRuntime().exec(shellString); + int exitValue = process.waitFor(); + if (0 != exitValue) { + logger.error("call shell failed. error code is :" + exitValue); + } + } catch (Throwable e) { + logger.error("call shell failed. " + e); + } + } + + /** + * 执行shell命令 + * + * @param shellString + */ + public static String callShellQuery(String shellString) { + logger.error("shellString:" + shellString); + BufferedReader reader = null; + String result = ""; + try { + Process process = Runtime.getRuntime().exec(shellString); + int exitValue = process.waitFor(); + if (0 != exitValue) { + logger.error("call shell failed. error code is :" + exitValue); + } + // 返回值 + reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = null; + while ((line = reader.readLine()) != null) { + result += line; + } + } catch (Throwable e) { + logger.error("call shell failed. " + e); + } + logger.info("call shell query result:" + result); + return result.trim(); + } + +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/application-dss.yml b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/application-dss.yml new file mode 100644 index 000000000..b0473838d --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/application-dss.yml @@ -0,0 +1,23 @@ + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + #instance: + #prefer-ip-address: true + #instance-id: ${spring.cloud.client.ip-address}:${server.port} + #metadata-map: + #test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss-guide-server.properties b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss-guide-server.properties new file mode 100644 index 000000000..ec2d5bf42 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss-guide-server.properties @@ -0,0 +1,39 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9210 +spring.spring.application.name=dss-guide-server +wds.linkis.server.version=v1 + +wds.linkis.log.clear=true + + + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.guide.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/guide/server/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.guide.server.entity +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.guide.server.dao + +#wds.linkis.gateway.ip=127.0.0.1 +#wds.linkis.gateway.port=9001 +#wds.linkis.gateway.url=http://127.0.0.1:9001/ + diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss.properties b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss.properties new file mode 100644 index 000000000..57b8354bb --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss.properties @@ -0,0 +1,41 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +wds.linkis.gateway.ip=xxxxxxx +wds.linkis.gateway.port=9001 +wds.linkis.gateway.url=http://xxxxxxx:9001/ +wds.linkis.gateway.wtss.url=http://xxxxxxx:9001/ + + +wds.linkis.mysql.is.encrypt=false +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://xxxxxxx:3306/dss_dev_1?characterEncoding=UTF-8 +wds.linkis.server.mybatis.datasource.username=xxxxxxx +wds.linkis.server.mybatis.datasource.password=xxxxxxx + +wds.linkis.server.version=v1 +spring.server.port=9210 +spring.spring.application.name=dss-guide-server + + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.guide.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/guide/server/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.guide.server.entity +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.guide.server.dao \ No newline at end of file diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/log4j.properties b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/log4j.properties new file mode 100644 index 000000000..f88f949da --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/log4j.properties @@ -0,0 +1,38 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/dss-data-governance-server.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/log4j2.xml b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/log4j2.xml new file mode 100644 index 000000000..c00e1eaa7 --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/log4j2.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/token.properties b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/token.properties new file mode 100644 index 000000000..a52a8983d --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/token.properties @@ -0,0 +1,19 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +${userName}=${password} \ No newline at end of file diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/scala/com/webank/wedatasphere/dss/guide/server/DSSGuideApplication.scala b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/scala/com/webank/wedatasphere/dss/guide/server/DSSGuideApplication.scala new file mode 100644 index 000000000..4518f5f2a --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/scala/com/webank/wedatasphere/dss/guide/server/DSSGuideApplication.scala @@ -0,0 +1,40 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.guide.server + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + +object DSSGuideApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-apps/dss-user-guide/pom.xml b/dss-apps/dss-user-guide/pom.xml new file mode 100644 index 000000000..9f1094218 --- /dev/null +++ b/dss-apps/dss-user-guide/pom.xml @@ -0,0 +1,18 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-user-guide + pom + + dss-user-guide-server + + + \ No newline at end of file diff --git a/dss-commons/dss-common/pom.xml b/dss-commons/dss-common/pom.xml new file mode 100644 index 000000000..b48afb1ab --- /dev/null +++ b/dss-commons/dss-common/pom.xml @@ -0,0 +1,176 @@ + + + + + 4.0.0 + + dss-commons + com.webank.wedatasphere.dss + 1.1.1 + + dss-common + + + 5.1.49 + 2.30.1 + + + + + org.apache.linkis + linkis-common + + + com.google.code.gson + gson + + + com.fasterxml.jackson.core + jackson-databind + + + org.codehaus.jackson + jackson-mapper-asl + + + org.scala-lang + scala-library + + + org.scala-lang + scala-compiler + + + org.scala-lang + scala-reflect + + + org.scala-lang + scalap + + + commons-lang + commons-lang + + + org.apache.linkis + linkis-label-common + ${linkis.version} + + + org.apache.linkis + linkis-protocol + ${linkis.version} + + + org.glassfish.jersey.media + jersey-media-json-jackson + ${jersey.version} + + + jackson-core + com.fasterxml.jackson.core + + + jackson-databind + com.fasterxml.jackson.core + + + + + org.glassfish.jersey.media + jersey-media-multipart + ${jersey.version} + + + org.glassfish.jersey.ext + jersey-entity-filtering + ${jersey.version} + + + + net.sourceforge.javacsv + javacsv + 2.0 + + + com.monitorjbl + xlsx-streamer + 1.2.1 + + + com.fasterxml.jackson.core + jackson-databind + + + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + mysql + mysql-connector-java + ${mysql.connector.version} + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + + product + + 1.0.0-SNAPSHOT + + + + + + + \ No newline at end of file diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/DSSCommonConf.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/DSSCommonConf.java new file mode 100644 index 000000000..d5ba12e37 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/DSSCommonConf.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class DSSCommonConf { + + public static final CommonVars DSS_IO_ENV = CommonVars.apply("wds.dss.server.io.env", "BDAP_DEV"); + + public static final CommonVars DSS_EXPORT_URL = CommonVars.apply("wds.dss.server.export.url", "/appcom/tmp/dss"); + + public static final CommonVars DSS_DOMAIN_LEVEL = CommonVars.apply("wds.linkis.gateway.domain.level", 3); + + /** + * The old value is bdp-user-ticket-id + */ + public static final CommonVars DSS_TOKEN_TICKET_KEY = CommonVars.apply("wds.dss.user.ticket.key", "linkis_user_session_ticket_id_v1"); + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/constant/project/ProjectUserPrivEnum.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/constant/project/ProjectUserPrivEnum.java new file mode 100644 index 000000000..73c3cb1f4 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/constant/project/ProjectUserPrivEnum.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.constant.project; + + +public enum ProjectUserPrivEnum { + + PRIV_ACCESS(1,"查看权限"), + PRIV_EDIT(2,"编辑权限"), + PRIV_RELEASE(3,"发布权限"); + + ProjectUserPrivEnum(int rank, String name) { + this.rank = rank; + this.name = name; + } + + private int rank; + + private String name; + + public int getRank() { + return rank; + } + + public void setRank(int rank) { + this.rank = rank; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/DSSWorkspace.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/DSSWorkspace.java new file mode 100644 index 000000000..9db02857d --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/DSSWorkspace.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +public interface DSSWorkspace { + + long getWorkspaceId(); + + String getWorkspaceName(); + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOEnv.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOEnv.java new file mode 100644 index 000000000..a4f8e248a --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOEnv.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +public enum IOEnv { + /** + * + */ + BDP_DEV,BDP_PRODUCTION,BDAP_DEV,BDAP_PRODUCTION +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOType.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOType.java new file mode 100644 index 000000000..5d7f23536 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOType.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +public enum IOType { + /** + * 区分工程导出和工作流导出 + * basepath/IO.properties + * type=PROJECT 或者 type=FLOE + */ + PROJECT,FLOW,ORCHESTRATOR +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/InputRelation.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/InputRelation.java new file mode 100644 index 000000000..a6a02a491 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/InputRelation.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +public class InputRelation { + + private Long id; + private String type; + private IOEnv sourceEnv; + private Long sourceID; + private IOEnv targetEnv; + private Long targetID; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public IOEnv getSourceEnv() { + return sourceEnv; + } + + public void setSourceEnv(IOEnv sourceEnv) { + this.sourceEnv = sourceEnv; + } + + public Long getSourceID() { + return sourceID; + } + + public void setSourceID(Long sourceID) { + this.sourceID = sourceID; + } + + public IOEnv getTargetEnv() { + return targetEnv; + } + + public void setTargetEnv(IOEnv targetEnv) { + this.targetEnv = targetEnv; + } + + public Long getTargetID() { + return targetID; + } + + public void setTargetID(Long targetID) { + this.targetID = targetID; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/PublishType.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/PublishType.java new file mode 100644 index 000000000..c871ab643 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/PublishType.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +public enum PublishType { + FULL,DELTA +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java new file mode 100644 index 000000000..c814c7b02 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +import java.io.Serializable; +import java.util.Objects; + +public class Resource implements Serializable { + private String fileName; + private String resourceId; + private String version; + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Resource resource = (Resource) o; + return resourceId.equals(resource.resourceId) && + version.equals(resource.version); + } + + @Override + public int hashCode() { + return Objects.hash(resourceId, version); + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java new file mode 100644 index 000000000..1d64eb759 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +public interface DSSEdge { + + String getSource(); + + void setSource(); + + String getTarget(); + + void setTarget(String target); + + String getSourceLocation(); + + void setSourceLocation(String sourceLocation); + + String getTargetLocation(); + + void setTargetLocation(String targetLocation); +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java new file mode 100644 index 000000000..c6c638916 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +public class DSSEdgeDefault implements DSSEdge { + private String source; + private String target; + private String sourceLocation; + private String targetLocation; + + @Override + public String getSource() { + return source; + } + + @Override + public void setSource() { + this.source = source; + } + + @Override + public String getTarget() { + return target; + } + + @Override + public void setTarget(String target) { + this.target = target; + } + + @Override + public String getSourceLocation() { + return sourceLocation; + } + + @Override + public void setSourceLocation(String sourceLocation) { + this.sourceLocation = sourceLocation; + } + + @Override + public String getTargetLocation() { + return targetLocation; + } + + @Override + public void setTargetLocation(String targetLocation) { + this.targetLocation = targetLocation; + } + + @Override + public String toString() { + return "DWSEdge{" + + "source='" + source + '\'' + + ", target='" + target + '\'' + + ", sourceLocation='" + sourceLocation + '\'' + + ", targetLocation='" + targetLocation + '\'' + + '}'; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java new file mode 100644 index 000000000..c37438e53 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +import com.webank.wedatasphere.dss.common.entity.Resource; + +import java.util.List; +import java.util.Map; + +public interface DSSNode extends Node { + + Layout getLayout(); + + void setLayout(Layout layout); + + Map getParams(); + + void setParams(Map params); + + List getResources(); + + void setResources(List resources); + + Map getJobContent(); + + void setJobContent(Map jobContent); + + String getUserProxy(); + + void setUserProxy(String userProxy); +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java new file mode 100644 index 000000000..72d24fefa --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java @@ -0,0 +1,158 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +import com.webank.wedatasphere.dss.common.entity.Resource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class DSSNodeDefault implements DSSNode { + private Layout layout; + private String id; + private String jobType; + private String lastUpdateTime; + private Map params; + private List resources; + private String title; + private String desc; + private String createTime; + private String userProxy; + /** + * dependencys 是该Node的依赖节点 + */ + private List dependencys = new ArrayList(); + /** + * jobContent 是node的内容 + */ + private Map jobContent; + + + @Override + public Layout getLayout() { + return layout; + } + + + @Override + public void setLayout(Layout layout) { + this.layout = layout; + } + + @Override + public Map getParams() { + return params; + } + + @Override + public void setParams(Map params) { + this.params = params; + } + + @Override + public List getResources() { + return resources; + } + + @Override + public void setResources(List resources) { + this.resources = resources; + } + + @Override + public Map getJobContent() { + return jobContent; + } + + @Override + public void setJobContent(Map jobContent) { + this.jobContent = jobContent; + } + + @Override + public String getUserProxy() { + return userProxy; + } + + @Override + public void setUserProxy(String userProxy) { + this.userProxy = userProxy; + } + + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Override + public String getNodeType() { + return jobType; + } + + @Override + public void setNodeType(String nodeType) { + this.jobType = nodeType; + } + + @Override + public String getName() { + return title; + } + + @Override + public void setName(String name) { + this.title = name; + } + + @Override + public void addDependency(String nodeName) { + dependencys.add(nodeName); + } + + @Override + public void setDependency(List dependency) { + this.dependencys = dependency; + } + + @Override + public void removeDependency(String nodeName) { + dependencys.remove(nodeName); + } + + @Override + public List getDependencys() { + return dependencys; + } + + @Override + public String toString() { + return "dwsNode{" + + "layout=" + layout + + ", id='" + id + '\'' + + ", jobType='" + jobType + '\'' + + ", lastUpdateTime=" + lastUpdateTime + + ", params=" + params + + ", resources=" + resources + + '}'; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java new file mode 100644 index 000000000..ac11c147a --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +public class Layout { + private float x; + private float y; + private float width; + private float height; + + public float getX() { + return x; + } + + public void setX(float x) { + this.x = x; + } + + public float getY() { + return y; + } + + public void setY(float y) { + this.y = y; + } + + public float getWidth() { + return width; + } + + public void setWidth(float width) { + this.width = width; + } + + public float getHeight() { + return height; + } + + public void setHeight(float height) { + this.height = height; + } + + @Override + public String toString() { + return "Layout{" + + "x=" + x + + ", y=" + y + + ", width=" + width + + ", height=" + height + + '}'; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java new file mode 100644 index 000000000..c3379329b --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + + +import java.util.List; + +/* + Node 接口 + */ +public interface Node { + String getId(); + + void setId(String id); + + String getNodeType(); + + void setNodeType(String nodeType); + + String getName(); + + void setName(String name); + + void addDependency(String nodeName); + + void setDependency(List dependency); + + void removeDependency(String nodeName); + + List getDependencys(); +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java new file mode 100644 index 000000000..7595a45ec --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java @@ -0,0 +1,174 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.project; + +import java.util.Date; + +public class DSSProject implements Project { + + private Long id; + private String name; + private String description; + private String createBy; + private String updateBy; + + private String username; + private Date createTime; + private Date updateTime; + //产品 + private String product; + //应用领域 + private Integer applicationArea; + //业务 + private String business; + private Integer workspaceId; + private String workspaceName; + + private Boolean editable; + + + @Override + public Long getId() { + return id; + } + + @Override + public void setId(Long id) { + this.id = id; + } + + @Override + public String getName() { + return name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public String getDescription() { + return description; + } + + @Override + public void setDescription(String description) { + this.description = description; + } + + @Override + public String getCreateBy() { + return createBy; + } + + @Override + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public Integer getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(Integer applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + @Override + public String toString() { + return "DSSProject{" + + "id=" + id + + ", name='" + name + '\'' + + ", description='" + description + '\'' + + ", createBy='" + createBy + '\'' + + ", username='" + username + '\'' + + ", createTime=" + createTime + + ", updateTime=" + updateTime + + ", product='" + product + '\'' + + ", applicationArea=" + applicationArea + + ", business='" + business + '\'' + + ", workspaceId=" + workspaceId + + ", workspaceName='" + workspaceName + '\'' + + ", editable=" + editable + + '}'; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java new file mode 100644 index 000000000..f9bd8bc69 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.project; + +public interface Project { + Long getId(); + + void setId(Long id); + + String getName(); + + void setName(String name); + + String getDescription(); + + void setDescription(String description); + + String getCreateBy(); + + void setCreateBy(String createBy); + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java new file mode 100644 index 000000000..572d5dd5d --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class DSSErrorException extends ErrorException { + + public DSSErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public DSSErrorException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java new file mode 100644 index 000000000..0cc1b6719 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +import org.apache.linkis.common.exception.WarnException; + +public class DSSRuntimeException extends WarnException { + + public DSSRuntimeException(String msg){ + super(100000, msg); + } + + public DSSRuntimeException(int errorCode, String msg) { + super(errorCode, msg); + } + + public DSSRuntimeException(int errorCode, String msg, Throwable e) { + super(errorCode, msg); + initCause(e); + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ErrorCode.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ErrorCode.java new file mode 100644 index 000000000..b7a45b557 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ErrorCode.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +public class ErrorCode { + + public static final int INVALID_PARAMS = 71001; + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java new file mode 100644 index 000000000..a305c0aef --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +@FunctionalInterface +public interface ThrowingConsumer { + void accept(T t) throws E; +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java new file mode 100644 index 000000000..5132bf453 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +@FunctionalInterface +public interface ThrowingFunction { + R accept(T t) throws E; +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabel.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabel.java new file mode 100644 index 000000000..c386c8de7 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabel.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import org.apache.linkis.manager.label.entity.GenericLabel; + +/** + * The basic label class of DSS. All other labels of DSS must extends this class. + */ +public class DSSLabel extends GenericLabel { +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabelUtil.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabelUtil.java new file mode 100644 index 000000000..a6ba4ff84 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabelUtil.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 生成DSSLabel工具类 + */ +public class DSSLabelUtil { + /** + * 临时构建标准labe map + */ + private static Map consltructLabelMap(String label) { + HashMap labelMap = new HashMap<>(); + labelMap.put(LabelKeyConvertor.ROUTE_LABEL_KEY, label); + return labelMap; + } + + /** + * 生成DSSLabel list + * @param label + * @return + */ + public static List createLabelList(String label){ + Map labelParam = consltructLabelMap(label); + LabelInstanceFactory labelInstanceFactory = new LabelInstanceFactory(labelParam); + List dssLabelList = labelInstanceFactory.getLabelList(); + return dssLabelList; + } + + /** + * 生成单一 DSSLabel + * @param label + * @return + */ + public static EnvDSSLabel createLabel(String label){ + Map labelParam = consltructLabelMap(label); + LabelInstanceFactory labelInstanceFactory = new LabelInstanceFactory(labelParam); + EnvDSSLabel envDSSLabel = labelInstanceFactory.getEnvDssLabel(); + return envDSSLabel; + } + + public static boolean isDevEnv(List dssLabels){ + for(DSSLabel dssLabel : dssLabels){ + String userEnv = null; + if(dssLabel instanceof EnvDSSLabel){ + userEnv = ((EnvDSSLabel)dssLabel).getEnv(); + }else{ + userEnv = dssLabel.getValue().get(dssLabel.getLabelKey()); + } + if(DSSCommonUtils.ENV_LABEL_VALUE_DEV.equalsIgnoreCase(userEnv)){ + return true; + } + } + return false; + } + + public static void main(String[] args) { + List labelList = DSSLabelUtil.createLabelList("dev"); + System.out.println(labelList.size()); + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/EnvDSSLabel.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/EnvDSSLabel.java new file mode 100644 index 000000000..4f513500b --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/EnvDSSLabel.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; +import org.apache.linkis.manager.label.entity.Feature; +import org.apache.linkis.manager.label.entity.annon.ValueSerialNum; +import java.util.HashMap; +import java.util.Map; + +/** + * Used to mark the env of this micro-services. + */ +public class EnvDSSLabel extends DSSLabel { + + public static final String DSS_ENV_LABEL_KEY = "DSSEnv"; + + public EnvDSSLabel(String env) { + setLabelKey(DSS_ENV_LABEL_KEY); + setEnv(env); + } + + public EnvDSSLabel() { + setLabelKey(DSS_ENV_LABEL_KEY); + } + + @Override + public Feature getFeature() { + return Feature.CORE; + } + + public String getEnv() { + if (null == getValue()) { + return null; + } + return getValue().get(DSS_ENV_LABEL_KEY); + } + + @ValueSerialNum(0) + public void setEnv(String env) { + if (null == getValue()) { + setValue(new HashMap<>(1)); + } + getValue().put(DSS_ENV_LABEL_KEY, env); + } + + public static void main(String[] args) { + Map map = new HashMap<>(); + map.put(DSS_ENV_LABEL_KEY, "dev"); + EnvDSSLabel label = (EnvDSSLabel) LabelBuilderFactoryContext.getLabelBuilderFactory().getLabels(map).get(0); + System.out.println("label: " + label.getEnv()); + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelInstanceFactory.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelInstanceFactory.java new file mode 100644 index 000000000..b9e5c255a --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelInstanceFactory.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import org.apache.linkis.manager.label.entity.Label; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +public class LabelInstanceFactory { + + private static final Logger log = LoggerFactory.getLogger(LabelInstanceFactory.class); + + private Map> labels = null; + + public LabelInstanceFactory(Map labelParam) { + labels = getLabels(labelParam); + } + + private Map> getLabels(Map originalLabels) { + Map originalLabelsMap = LabelKeyConvertor.labelKeyConvert(originalLabels); + return LabelBuilder.buildLabel(originalLabelsMap); + } + + public Map getLabelsMap() { + HashMap resultLabels = new HashMap<>(); + labels.entrySet().stream().map(item -> resultLabels.put(item.getKey(), (DSSLabel) item.getValue())); + return resultLabels; + } + + public List getLabelList() { + List dssLabelList = new ArrayList<>(); + labels.entrySet().stream().map(item -> dssLabelList.add((DSSLabel) (item.getValue()))); + return dssLabelList; + } + + public EnvDSSLabel getEnvDssLabel() { + return (EnvDSSLabel) labels.get(EnvDSSLabel.DSS_ENV_LABEL_KEY); + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelKeyConvertor.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelKeyConvertor.java new file mode 100644 index 000000000..e80268ce8 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelKeyConvertor.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +public class LabelKeyConvertor { + + private static final Logger log = LoggerFactory.getLogger(LabelKeyConvertor.class); + + + public static String ROUTE_LABEL_KEY = "route"; + + private static final Map convertedlabels = new HashMap<>(); + + static { + convertedlabels.put(ROUTE_LABEL_KEY, EnvDSSLabel.DSS_ENV_LABEL_KEY); + } + + private static String getRealLabelKey(String originalLabelKey) { + return convertedlabels.get(originalLabelKey); + } + + public static Map labelKeyConvert(Map labels) { + Map finalLabels = new HashMap<>(); + for (Map.Entry item : labels.entrySet()) { + String realLabelKey = LabelKeyConvertor.getRealLabelKey(item.getKey()); + finalLabels.put(realLabelKey, item.getValue()); + } + log.info("Get labels map: {}", finalLabels); + return finalLabels; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelRouteVO.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelRouteVO.java new file mode 100644 index 000000000..482451c70 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelRouteVO.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +public class LabelRouteVO { + + private String route; + + public String getRoute() { + return route; + } + + public void setRoute(String route) { + this.route = route; + } + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/JobStatus.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/JobStatus.java new file mode 100644 index 000000000..8409ea1f6 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/JobStatus.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol; + +public enum JobStatus { + + /** + * JobStatus是用来进行发布任务的状态的追踪 + */ + + Inited("Inited", 1), + Running("Running", 2), + Success("Success", 3), + Failed("Failed", 4); + + private String status; + private int index; + + private JobStatus(String status, int index){ + this.status = status; + this.index = index; + } + + public static boolean isSuccess(String status){ + return Success.status.equals(status); + } + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoRequest.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoRequest.java new file mode 100644 index 000000000..c0977fb09 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoRequest.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol.project; + +public class ProjectInfoRequest { + + private long projectId; + + public long getProjectId() { + return projectId; + } + + public void setProjectId(long projectId) { + this.projectId = projectId; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRefIdRequest.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRefIdRequest.java new file mode 100644 index 000000000..ff0beb1f9 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRefIdRequest.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.common.protocol.project; + +public class ProjectRefIdRequest { + private Long appInstanceId; + private Long dssProjectId; + + public ProjectRefIdRequest(Long appInstanceId, Long dssProjectId) { + this.appInstanceId = appInstanceId; + this.dssProjectId = dssProjectId; + } + + public Long getAppInstanceId() { + return appInstanceId; + } + + public void setAppInstanceId(Long appInstanceId) { + this.appInstanceId = appInstanceId; + } + + public Long getDssProjectId() { + return dssProjectId; + } + + public void setDssProjectId(Long dssProjectId) { + this.dssProjectId = dssProjectId; + } + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRefIdResponse.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRefIdResponse.java new file mode 100644 index 000000000..92f33288c --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRefIdResponse.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.dss.common.protocol.project; + +public class ProjectRefIdResponse { + private Long appInstanceId; + private Long dssProjectId; + private Long refProjectId; + + public ProjectRefIdResponse(Long appInstanceId, Long dssProjectId, Long refProjectId) { + this.appInstanceId = appInstanceId; + this.dssProjectId = dssProjectId; + this.refProjectId = refProjectId; + } + + public Long getAppInstanceId() { + return appInstanceId; + } + + public void setAppInstanceId(Long appInstanceId) { + this.appInstanceId = appInstanceId; + } + + public Long getDssProjectId() { + return dssProjectId; + } + + public void setDssProjectId(Long dssProjectId) { + this.dssProjectId = dssProjectId; + } + + public Long getRefProjectId() { + return refProjectId; + } + + public void setRefProjectId(Long refProjectId) { + this.refProjectId = refProjectId; + } + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationRequest.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationRequest.java new file mode 100644 index 000000000..5f2024c0e --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationRequest.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol.project; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; + +public class ProjectRelationRequest { + + private Long dssProjectId; + + private String appConnName; + + private List dssLabels; + + public ProjectRelationRequest(Long dssProjectId, String appconnName, List dssLabels) { + this.dssProjectId = dssProjectId; + this.appConnName = appconnName; + this.dssLabels = dssLabels; + } + + public Long getDssProjectId() { + return dssProjectId; + } + + public void setDssProjectId(Long dssProjectId) { + this.dssProjectId = dssProjectId; + } + + public String getAppconnName() { + return appConnName; + } + + public void setAppconnName(String appconnName) { + this.appConnName = appconnName; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationResponse.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationResponse.java new file mode 100644 index 000000000..590edb922 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationResponse.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol.project; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; + +public class ProjectRelationResponse { + + private Long dssProjectId; + + private String appConnName; + + private List dssLabels; + + private Long appInstanceProjectId; + + public ProjectRelationResponse(Long dssProjectId, String appConnName, List dssLabels, Long appInstanceProjectId) { + this.dssProjectId = dssProjectId; + this.appConnName = appConnName; + this.dssLabels = dssLabels; + this.appInstanceProjectId = appInstanceProjectId; + } + + public Long getDssProjectId() { + return dssProjectId; + } + + public void setDssProjectId(Long dssProjectId) { + this.dssProjectId = dssProjectId; + } + + public String getAppConnName() { + return appConnName; + } + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public Long getAppInstanceProjectId() { + return appInstanceProjectId; + } + + public void setAppInstanceProjectId(Long appInstanceProjectId) { + this.appInstanceProjectId = appInstanceProjectId; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectUserAuthRequest.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectUserAuthRequest.java new file mode 100644 index 000000000..d8ba46188 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectUserAuthRequest.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.common.protocol.project; + +public class ProjectUserAuthRequest { + private Long projectId; + private String userName; + + public ProjectUserAuthRequest(Long projectId, String userName) { + this.projectId = projectId; + this.userName = userName; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectUserAuthResponse.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectUserAuthResponse.java new file mode 100644 index 000000000..cc07d33ac --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectUserAuthResponse.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.dss.common.protocol.project; + +import java.util.List; + +public class ProjectUserAuthResponse { + private Long projectId; + private String userName; + + private List privList; + + private String projectOwner; + + public ProjectUserAuthResponse(Long projectId, String userName, List privList, String projectOwner) { + this.projectId = projectId; + this.userName = userName; + this.privList = privList; + this.projectOwner = projectOwner; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public List getPrivList() { + return privList; + } + + public void setPrivList(List privList) { + this.privList = privList; + } + + public String getProjectOwner() { + return projectOwner; + } + + public void setProjectOwner(String projectOwner) { + this.projectOwner = projectOwner; + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ClassUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ClassUtils.java new file mode 100644 index 000000000..11ec9e7f5 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ClassUtils.java @@ -0,0 +1,129 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.reflections.Reflections; + +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class ClassUtils { + + private static final ClassHelper CLASS_HELPER = new ClassHelper() { + @Override + protected Reflections getReflections(Class clazz) { + return org.apache.linkis.common.utils.ClassUtils.reflections(); + } + }; + + public static T getInstance(Class clazz) throws DSSErrorException { + return CLASS_HELPER.getInstance(clazz); + } + + public static T getInstanceOrWarn(Class clazz) { + return CLASS_HELPER.getInstanceOrWarn(clazz); + } + + public static T getInstanceOrDefault(Class clazz, T defaultValue) { + return CLASS_HELPER.getInstanceOrDefault(clazz, defaultValue); + } + + public static T getInstanceOrDefault(Class clazz, Predicate> filterOp, T defaultValue) { + return CLASS_HELPER.getInstanceOrDefault(clazz, filterOp, defaultValue); + } + + public static List getInstances(Class clazz) { + return CLASS_HELPER.getInstances(clazz); + } + + public static List getInstances(Class clazz, Predicate> filterOp) { + return CLASS_HELPER.getInstances(clazz, filterOp); + } + + public static List> getClasses(Class clazz) { + return CLASS_HELPER.getClasses(clazz); + } + + public abstract static class ClassHelper { + + protected abstract Reflections getReflections(Class clazz); + + public T getInstance(Class clazz) throws DSSErrorException { + List> factoryClasses = getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !org.apache.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c)).collect(Collectors.toList()); + if(factoryClasses.isEmpty()) { + DSSExceptionUtils.dealErrorException(60053, "Cannot find instance for " + clazz.getSimpleName(), DSSErrorException.class); + } else if(factoryClasses.size() > 1) { + DSSExceptionUtils.dealErrorException(60053, "Too many instances for " + clazz.getSimpleName() + ", exists: " + factoryClasses, + DSSErrorException.class); + } + T t = null; + try { + t = factoryClasses.get(0).newInstance(); + } catch (Exception e) { + DSSExceptionUtils.dealErrorException(60053, "Instance " + clazz.getSimpleName() + " failed", e, DSSErrorException.class); + } + return t; + } + + public T getInstanceOrWarn(Class clazz) { + try { + return getInstance(clazz); + } catch (DSSErrorException e) { + DSSExceptionUtils.dealWarnException(60053, e.getDesc(), ExceptionUtils.getCause(e), DSSRuntimeException.class); + } + return null; + } + + public T getInstanceOrDefault(Class clazz, T defaultValue) { + Optional optional = getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !org.apache.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c) && + !c.isInstance(defaultValue)).findFirst().map(DSSExceptionUtils.map(Class::newInstance)); + return optional.orElse(defaultValue); + } + + public T getInstanceOrDefault(Class clazz, Predicate> filterOp, T defaultValue) { + Optional optional = getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !org.apache.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c) && + filterOp.test(c)).findFirst().map(DSSExceptionUtils.map(Class::newInstance)); + return optional.orElse(defaultValue); + } + + public List getInstances(Class clazz) { + return getInstances(clazz, c -> true); + } + + public List getInstances(Class clazz, Predicate> filterOp) { + return getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !org.apache.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c) && filterOp.test(c)) + .map(DSSExceptionUtils.map(Class::newInstance)).collect(Collectors.toList()); + } + + public List> getClasses(Class clazz) { + return getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !org.apache.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c)) + .collect(Collectors.toList()); + } + + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java new file mode 100644 index 000000000..b51032b54 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java @@ -0,0 +1,79 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.JsonUtils; + +public class DSSCommonUtils { + + public static final String FLOW_RESOURCE_NAME = "resources"; + + public static final String FLOW_EDGES_NAME = "edges"; + + public static final String FLOW_PARENT_NAME = "parent"; + + public static final String NODE_RESOURCE_NAME = "resources"; + + public static final String FLOW_NODE_NAME = "nodes"; + + public static final String FLOW_PROP_NAME = "props"; + + public static final String NODE_PROP_NAME = "params"; + + + public static final String NODE_ID_NAME = "id"; + + public static final String NODE_NAME_NAME = "title"; + + public static final Gson COMMON_GSON = new GsonBuilder().disableHtmlEscaping().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls() + .registerTypeAdapter(Double.class, (JsonSerializer) (t, type, jsonSerializationContext) -> { + if (t == t.longValue()) { + return new JsonPrimitive(t.longValue()); + } else { + return new JsonPrimitive(t); + } + }).create(); + + public static final ObjectMapper JACKSON = JsonUtils.jackson(); + + public static final String ENV_LABEL_VALUE_DEV = "dev"; + + public static final String DSS_LABELS_KEY = "labels"; + + public static final CommonVars DSS_HOME = CommonVars.apply("DSS_HOME", ""); + + public static long parseToLong(Object val) { + if (val instanceof Double) { + return ((Double) val).longValue(); + } else if (val instanceof Integer) { + return new Double((Integer) val).longValue(); + } else if (val instanceof Long) { + return (Long) val; + } else if (val != null) { + return Long.parseLong(val.toString()); + } + throw new DSSRuntimeException(90322, "parse the return of externalSystem failed, the value is null."); + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java new file mode 100644 index 000000000..3660a1d2f --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java @@ -0,0 +1,124 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.exception.ThrowingConsumer; +import com.webank.wedatasphere.dss.common.exception.ThrowingFunction; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.WarnException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; +import java.util.function.Consumer; +import java.util.function.Function; + +public class DSSExceptionUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSExceptionUtils.class); + + public static Consumer handling( + ThrowingConsumer throwingConsumer) { + return i -> { + try { + throwingConsumer.accept(i); + } catch (Exception e) { + LOGGER.error("execute failed, reason: ", e); + throw new DSSRuntimeException(ExceptionUtils.getRootCauseMessage(e)); + } + }; + } + + public static Function map( + ThrowingFunction throwingFunction) { + return i -> { + try { + return throwingFunction.accept(i); + } catch (Exception e) { + LOGGER.error("execute failed,reason:", e); + throw new DSSRuntimeException(ExceptionUtils.getRootCauseMessage(e)); + } + }; + } + + public static R tryAndWarn( + ThrowingFunction throwingFunction) { + Function function = i -> { + try { + return throwingFunction.accept(i); + } catch (Exception e) { + LOGGER.error("execute failed,reason:", e); + throw new DSSRuntimeException(53320, ExceptionUtils.getRootCauseMessage(e)); + } + }; + return function.apply(null); + } + + public static void dealErrorException(int errorCode, String errorDesc, Throwable throwable, + Class clazz) throws T{ + T errorException; + try { + Constructor constructor = clazz.getConstructor(int.class, String.class); + errorException = constructor.newInstance(errorCode, errorDesc); + } catch (Exception e) { + if(throwable == null) { + throw new DSSRuntimeException(errorCode, errorDesc, e); + } + throw new DSSRuntimeException(errorCode, errorDesc, throwable); + } + errorException.setErrCode(errorCode); + errorException.setDesc(errorDesc); + if(throwable != null) { + errorException.initCause(throwable); + } + throw errorException; + } + + + public static void dealErrorException(int errorCode, String errorDesc, + Class clazz) throws T { + dealErrorException(errorCode, errorDesc, null, clazz); + } + + public static void dealWarnException(int errorCode, String errorDesc, + Class clazz) { + dealWarnException(errorCode, errorDesc, null, clazz); + } + + public static void dealWarnException(int errorCode, String errorDesc, Throwable throwable, + Class clazz) { + T warnException; + try { + Constructor constructor = clazz.getConstructor(int.class, String.class); + warnException = constructor.newInstance(errorCode, errorDesc); + } catch (Exception e) { + if(throwable == null) { + throw new DSSRuntimeException(errorCode, errorDesc, e); + } + throw new DSSRuntimeException(errorCode, errorDesc, throwable); + } + warnException.setErrCode(errorCode); + warnException.setDesc(errorDesc); + if(throwable != null) { + warnException.initCause(throwable); + } + throw warnException; + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java new file mode 100644 index 000000000..195265097 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.Arrays; +import java.util.List; + +public class FileHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileHelper.class); + + public static boolean checkDirExists(String dir) { + File file = new File(dir); + return file.exists() && file.isDirectory(); + } + + /** + * 获取某个文件夹路径下所有文件的全路径名 + * + * @param path 文件夹路径 + * @param fileNameList 所有文件全路径名称 + */ + public static void getAllFileNames(String path, List fileNameList) { + File file = new File(path); + File[] files = file.listFiles(); + String[] names = file.list(); + if (names != null) { + String[] completeNames = new String[names.length]; + for (int i = 0; i < names.length; i++) { + completeNames[i] = path + names[i]; + } + fileNameList.addAll(Arrays.asList(completeNames)); + } + for (File tmpFile : files) { + //如果文件夹下有子文件夹,获取子文件夹下的所有文件全路径。 + if (tmpFile.isDirectory()) { + getAllFileNames(tmpFile.getAbsolutePath() + File.separator, fileNameList); + } + } + } + + /** + * 读取文本文件 + * + * @param filename 文件全路径 + */ + public static String readFile(String filename) { + StringBuffer fileContent = new StringBuffer(); + try (FileInputStream fis = new FileInputStream(filename); + BufferedInputStream bufferedInput = new BufferedInputStream(fis)) { + int bytesRead = 0; + byte[] buffer = new byte[1024]; + while ((bytesRead = bufferedInput.read(buffer)) != -1) { + String tmpContent = new String(buffer, 0, bytesRead); + fileContent.append(tmpContent); + } + } catch (FileNotFoundException ex) { + LOGGER.error(",文件路径不存在,文件路径:{}", filename, ex); + } catch (IOException e) { + LOGGER.error("读取文件失败,文件路径:{}", filename, e); + } + return fileContent.toString(); + } +} \ No newline at end of file diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java new file mode 100644 index 000000000..d73d9d51d --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java @@ -0,0 +1,116 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import com.webank.wedatasphere.dss.common.conf.DSSCommonConf; +import com.webank.wedatasphere.dss.common.entity.IOEnv; +import com.webank.wedatasphere.dss.common.entity.IOType; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.Properties; + +public class IoUtils { + + private static Logger logger = LoggerFactory.getLogger(IoUtils.class); + private static final String DATE_FORMAT = "yyyyMMddHHmmssSSS"; + private static final String DEFAULT_IO_FILE_NAME = "IO.properties"; + + public static String generateIOPath(String userName,String projectName,String subDir) { + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(baseUrl, dataStr, userName, projectName, subDir); + } + + private static String addFileSeparator(String... str) { + return Arrays.stream(str).reduce((a, b) -> a + File.separator + b).orElse(""); + } + + public static OutputStream generateExportOutputStream(String path) throws IOException { + File file = new File(path); + if (!file.getParentFile().exists()) { + FileUtils.forceMkdir(file.getParentFile()); + } + if (file.exists()) { + logger.warn(String.format("%s is exist,delete it", path)); + file.delete(); + } + file.createNewFile(); + return FileUtils.openOutputStream(file, true); + } + + public static InputStream generateInputInputStream(String path)throws IOException{ + return new FileInputStream(path); + } + + public static void generateIOType(IOType ioType, String basePath) throws IOException { + generateIOProperties("type",ioType.name(),basePath); + } + + public static void generateIOProperties(String key,String value,String basePath) throws IOException { + Properties properties = new Properties(); + properties.setProperty(key,value); + + File file = new File(basePath + File.separator + DEFAULT_IO_FILE_NAME); + if (!file.getParentFile().exists()) { + FileUtils.forceMkdir(file.getParentFile()); + } + if(!file.exists()){ + file.createNewFile(); + } + try(FileOutputStream fileOutputStream = FileUtils.openOutputStream(file, true)) { + properties.store(fileOutputStream,""); + } + + } + + public static String readIOProperties(String key,String basepath) throws IOException { + try(FileInputStream inputStream = new FileInputStream(basepath + File.separator + DEFAULT_IO_FILE_NAME)){ + Properties properties = new Properties(); + properties.load(inputStream); + return properties.get(key).toString(); + } + + } + + public static void generateIOEnv(String basePath) throws IOException { + generateIOProperties("env",getDSSServerEnv().name(),basePath); + } + + public static IOEnv getDSSServerEnv(){ + //dssserverEnv 是当前dss服务启动的env环境 + return IOEnv.valueOf(DSSCommonConf.DSS_IO_ENV.getValue()); + } + + public static String addVersion(String version){ + String num = String.valueOf(Integer.valueOf(version.substring(1)) + 1); + int length = num.length(); + return version.substring(0,version.length() - length) + num; + } + + public static String subVersion(String version){ + String num = String.valueOf(Integer.valueOf(version.substring(1)) - 1); + int length = num.length(); + return version.substring(0,version.length() - length) + num; + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/MapUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/MapUtils.java new file mode 100644 index 000000000..06cba7321 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/MapUtils.java @@ -0,0 +1,63 @@ +package com.webank.wedatasphere.dss.common.utils; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-12 + * @since 0.5.0 + */ +public class MapUtils { + + public static boolean isEmpty(Map map) { + return map == null || map.isEmpty(); + } + + public static boolean isNotEmpty(Map map) { + return !isEmpty(map); + } + + public static Map newMap(K key, V value) { + Map map = new HashMap<>(1); + map.put(key, value); + return map; + } + + public static Map newMap(K key1, V value1, K key2, V value2) { + Map map = new HashMap<>(2); + map.put(key1, value1); + map.put(key2, value2); + return map; + } + + public static MapBuilder newMapBuilder() { + return new MapBuilder(); + } + + public static Map newCommonMap(String key, Object value) { + return newMap(key, value); + } + + public static Map newCommonMap(String key1, Object value1, String key2, Object value2) { + return newMap(key1, value1, key2, value2); + } + + public static MapBuilder newCommonMapBuilder() { + return new MapBuilder(); + } + + public static class MapBuilder { + + private Map map = new HashMap<>(); + + public MapBuilder put(K key, V value) { + map.put(key, value); + return this; + } + + public Map build() { + return map; + } + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java new file mode 100644 index 000000000..fcc31ea1b --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java @@ -0,0 +1,192 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + + + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +public class ZipHelper { + + private static final Logger logger = LoggerFactory.getLogger(ZipHelper.class); + + private static final String ZIP_CMD = "zip"; + private static final String UN_ZIP_CMD = "unzip"; + private static final String RECURSIVE = "-r"; + private static final String ZIP_TYPE = ".zip"; + + public static String zip(String dirPath) throws DSSErrorException { + return zip(dirPath, true); + } + + /** + * ZipHelper可以将传入的path进行打包 + * @param dirPath 需要打包的project路径,绝对路径 + * @return 打包之后的zip包全路径 + */ + public static String zip(String dirPath, boolean deleteOriginDir)throws DSSErrorException { + if(!FileHelper.checkDirExists(dirPath)){ + logger.error("{} 不存在, 不能创建zip文件", dirPath); + throw new DSSErrorException(90001,dirPath + " does not exist, can not zip"); + } + //先用简单的方法,调用新进程进行压缩 + String[] strArr = dirPath.split(File.separator); + String shortPath = strArr[strArr.length - 1]; + String workPath = dirPath.substring(0, dirPath.length() - shortPath.length() - 1); + List list = new ArrayList<>(); + list.add(ZIP_CMD); + list.add(RECURSIVE); + String zipFilePath = shortPath + ZIP_TYPE; + String longZipFilePath = dirPath + ZIP_TYPE; + list.add(zipFilePath); + list.add(shortPath); + ProcessBuilder processBuilder = new ProcessBuilder(list); + processBuilder.redirectErrorStream(true); + processBuilder.directory(new File(workPath)); + BufferedReader infoReader = null; + BufferedReader errorReader = null; + try{ + Process process = processBuilder.start(); + infoReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String infoLine = null; + while((infoLine = infoReader.readLine()) != null){ + logger.info("process output: {} ", infoLine); + } + String errorLine = null; + StringBuilder errMsg = new StringBuilder(); + while((errorLine = errorReader.readLine()) != null){ + if (StringUtils.isNotEmpty(errorLine)){ + errMsg.append(errorLine).append("\n"); + } + logger.error("process error: {} ", errorLine); + } + int exitCode = process.waitFor(); + if (exitCode != 0){ + throw new DSSErrorException(90002,errMsg.toString()); + } + }catch(final Exception e){ + logger.error("{} 压缩成 zip 文件失败, reason: ", e); + DSSErrorException exception = new DSSErrorException(90003,dirPath + " to zip file failed"); + exception.initCause(e); + throw exception; + } finally { + //删掉整个目录 + IOUtils.closeQuietly(infoReader); + IOUtils.closeQuietly(errorReader); + if(deleteOriginDir) { + File file = new File(dirPath); + logger.info("生成zip文件{}",longZipFilePath); + logger.info("开始删除目录 {}", dirPath); + if (deleteDir(file)){ + logger.info("结束删除目录 {} 成功", dirPath); + }else{ + logger.info("删除目录 {} 失败", dirPath); + } + } + } + return longZipFilePath; + } + + + + public static String unzip(String dirPath)throws DSSErrorException { + File file = new File(dirPath); + if(!file.exists()){ + logger.error("{} 不存在, 不能解压zip文件", dirPath); + throw new DSSErrorException(90001,dirPath + " does not exist, can not unzip"); + } + //先用简单的方法,调用新进程进行压缩 + String[] strArr = dirPath.split(File.separator); + String shortPath = strArr[strArr.length - 1]; + String workPath = dirPath.substring(0, dirPath.length() - shortPath.length() - 1); + List list = new ArrayList<>(); + list.add(UN_ZIP_CMD); + String longZipFilePath = dirPath.replace(ZIP_TYPE,""); + list.add(shortPath); + ProcessBuilder processBuilder = new ProcessBuilder(list); + processBuilder.redirectErrorStream(true); + processBuilder.directory(new File(workPath)); + BufferedReader infoReader = null; + BufferedReader errorReader = null; + try{ + Process process = processBuilder.start(); + infoReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String infoLine = null; + while((infoLine = infoReader.readLine()) != null){ + logger.info("process output: {} ", infoLine); + } + String errorLine = null; + StringBuilder errMsg = new StringBuilder(); + while((errorLine = errorReader.readLine()) != null){ + if (StringUtils.isNotEmpty(errorLine)){ + errMsg.append(errorLine).append("\n"); + } + logger.error("process error: {} ", errorLine); + } + int exitCode = process.waitFor(); + if (exitCode != 0){ + throw new DSSErrorException(90007,errMsg.toString()); + } + }catch(final Exception e){ + logger.error("{} 解压缩 zip 文件失败, reason: ", e); + DSSErrorException exception = new DSSErrorException(90009,dirPath + " to zip file failed"); + exception.initCause(e); + throw exception; + } finally { + + logger.info("生成解压目录{}", longZipFilePath); + IOUtils.closeQuietly(infoReader); + IOUtils.closeQuietly(errorReader); + } + return longZipFilePath; + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + if (children != null && children.length > 0){ + for (String s : children) { + boolean success = deleteDir(new File(dir, s)); + if (!success) { + return false; + } + } + } + } + // 目录此时为空,可以删除 + return dir.delete(); + } + + public static String zipExportProject(String projectPath) throws DSSErrorException { + if(projectPath.endsWith(File.separator)) { + projectPath = projectPath.substring(0, projectPath.lastIndexOf(File.separator)); + } + return ZipHelper.zip(projectPath); + } +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/RestfulCatchAOP.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/RestfulCatchAOP.scala new file mode 100644 index 000000000..0aa92d7b2 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/RestfulCatchAOP.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.dss.common + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.server.{Message, catchIt} +import org.aspectj.lang.ProceedingJoinPoint +import org.aspectj.lang.annotation.{Around, Aspect, Pointcut} +import org.springframework.stereotype.Component + +@Aspect +@Component +class RestfulCatchAOP extends Logging { + + @Pointcut("@annotation(javax.ws.rs.Path) && execution(public org.apache.linkis.server.Message *(..))") + def restfulMessageCatch(): Unit = {} + + @Around("restfulMessageCatch()") + def dealMessageRestful(proceedingJoinPoint: ProceedingJoinPoint): Object = catchIt { + proceedingJoinPoint.proceed().asInstanceOf[Message] + } + + @Pointcut("@annotation(javax.ws.rs.Path) && execution(public javax.ws.rs.core.Response *(..)))") + def restfulResponseCatch(): Unit = {} + + @Around("restfulResponseCatch()") + def dealResponseRestful(proceedingJoinPoint: ProceedingJoinPoint): Object = { + val resp: Message = catchIt { + Message + return proceedingJoinPoint.proceed() + } + resp + } +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/label/LabelBuilder.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/label/LabelBuilder.scala new file mode 100644 index 000000000..6982464fe --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/label/LabelBuilder.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label + +import org.apache.linkis.manager.label.entity.Label + +import scala.collection.JavaConverters._ +import java.util + +import org.apache.linkis.manager.label.builder.factory.{LabelBuilderFactory, LabelBuilderFactoryContext} + +object LabelBuilder { + + def buildLabel(labelMap: util.Map[String, Object]): util.Map[String, Label[_]] = { + val labelKeyValueMap = new util.HashMap[String, Label[_]]() + if (null != labelMap && !labelMap.isEmpty) { + val list: util.List[Label[_]] = labelBuilderFactoryProxyMethod.getLabels(labelMap.asInstanceOf[util.Map[String, AnyRef]]) + if (null != list) { + list.asScala.filter(_ != null).foreach { + label => labelKeyValueMap.put(label.getLabelKey, label) + } + } + } + labelKeyValueMap + } + + private def labelBuilderFactoryProxyMethod: LabelBuilderFactory = { + LabelBuilderFactoryContext.getLabelBuilderFactory + } +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocol.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocol.scala new file mode 100644 index 000000000..ad17a1ba3 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocol.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol + +trait DSSProtocol { + +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala new file mode 100644 index 000000000..b9da4e5c6 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala @@ -0,0 +1,63 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol + +import com.webank.wedatasphere.dss.common.label.DSSLabel + +case class ResponseCreateOrchestrator(orchestratorId: Long, + orchestratorVersionId: Long) + +case class ResponseExportOrchestrator(resourceId: String, + version: String, + orcVersionId: Long) + +case class ResponseImportOrchestrator(orcId: Long) + +case class RequestUpdateWorkflow(userName: String, + flowID: Long, + flowName: String, + description: String, + uses: String) + +case class RequestDeleteWorkflow(userName: String, flowID: Long) + + +case class RequestExportWorkflow(userName: String, + flowID: Long, + projectId: Long, + projectName: String, + workspaceStr: String, + dssLabelList: java.util.List[DSSLabel] + ) + +case class ResponseExportWorkflow(resourceId: String, version: String, flowID: Long) + +case class RequestQueryWorkFlow(userName: String, rootFlowId: Long) + +case class ProxyUserCheckRequest(userName: String, proxyUser: String) + +case class ResponseProxyUserCheck(canProxy: Boolean,proxyUserList:java.util.List[String]) + + + + + + + + + + diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/framework/ReleaseOrchestratorRequest.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/framework/ReleaseOrchestratorRequest.scala new file mode 100644 index 000000000..3e89719c0 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/framework/ReleaseOrchestratorRequest.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol.framework + +import com.webank.wedatasphere.dss.common.protocol.DSSProtocol + +trait FrameworkProtocol extends DSSProtocol + +case class ReleaseOrchestratorRequest(releaseUser:String, orchestratorVersionId:Long, orchestratorId:Long, + dssLabel:String, workspaceName:String, workSpaceStr: String) extends FrameworkProtocol + +case class ReleaseOrchestratorResponse(jobId:Long) extends FrameworkProtocol + +case class ReleaseOrchestratorStatusRequest(jobId:Long) extends FrameworkProtocol + +case class ReleaseOrchestratorStatusResponse(status:String, jobId:Long, errorMsg:String) extends FrameworkProtocol + + + + + + diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/DSSMainHelper.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/DSSMainHelper.scala new file mode 100644 index 000000000..0601fae68 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/DSSMainHelper.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils + +import org.apache.linkis.common.conf.{BDPConfiguration, CommonVars, DSSConfiguration} + +import scala.collection.JavaConversions._ + + +object DSSMainHelper { + +// val DSS_DEFAULT_PROPERTY_FILE_NAME = CommonVars("wds.dss.default.property.file", "dss.properties") +// val DSS_SERVER_DEFAULT_PROPERTY_FILE_NAME = CommonVars("wds.dss.server.default.property.file", "%s.properties") + + def formatPropertyFiles(serviceName: String): Unit = { + sys.props.put("wds.linkis.configuration", "dss.properties") + sys.props.put("wds.linkis.server.conf", serviceName+".properties") + } + + //TODO wait for linkis re-written + @Deprecated + def addExtraPropertyFiles(filePaths: String *): Unit = { + sys.props.put("wds.linkis.server.confs", filePaths.mkString(",")) + } + + def getExtraSpringOptions: Array[String] = { + "--spring.profiles.active=dss" +: DSSConfiguration.getAllProperties.filter { case (k, v) => k != null && k.startsWith("spring.")} + .map{ case (k, v) => { + val realKey=k.substring(7) + s"--$realKey=$v" + } + }.toArray + } + +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/MessageUtils.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/MessageUtils.scala new file mode 100644 index 000000000..27d979d80 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/MessageUtils.scala @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.common.utils + +import org.apache.linkis.server.Message + +import javax.ws.rs.core.Response + +object MessageUtils { + implicit def messageToResponse(message: Message): Response = + Response.status(messageToHttpStatus(message)).entity(message).build() + + //implicit def responseToMessage(response: Response): Message = response.readEntity(classOf[Message]) + + def messageToHttpStatus(message: Message): Int = message.getStatus match { + case -1 => 401 + case 0 => 200 + case 1 => 400 + case 2 => 412 + case 3 => 403 + case 4 => 206 + } +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/VariableUtils.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/VariableUtils.scala new file mode 100644 index 000000000..63c72ca19 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/VariableUtils.scala @@ -0,0 +1,757 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils + +import java.text.SimpleDateFormat +import java.util +import java.util.concurrent.{ExecutorService, Executors} +import java.util.{Calendar, Date} + +import org.apache.linkis.common.exception.LinkisCommonErrorException +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.commons.lang.StringUtils +import org.apache.commons.lang.time.DateUtils + +import scala.collection.mutable +import scala.collection.convert.WrapAsScala._ +import scala.util.matching.Regex + +object VariableUtils extends Logging { + + val RUN_DATE = "run_date" + val RUN_TODAY = "run_today" + val RUN_MON = "run_mon" + + val dateFormatLocal = new ThreadLocal[SimpleDateFormat]() { + override protected def initialValue = new SimpleDateFormat("yyyyMMdd") + } + + val dateFormatStdLocal = new ThreadLocal[SimpleDateFormat]() { + override protected def initialValue = new SimpleDateFormat("yyyy-MM-dd") + } + + val dateFormatMonLocal = new ThreadLocal[SimpleDateFormat]() { + override protected def initialValue = new SimpleDateFormat("yyyyMM") + } + + val dateFormatMonStdLocal = new ThreadLocal[SimpleDateFormat]() { + override protected def initialValue = new SimpleDateFormat("yyyy-MM") + } + + val codeReg: Regex = "\\$\\{\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*[\\+\\-\\*/]?\\s*[A-Za-z0-9_\\.]*\\s*\\}".r + val calReg: Regex = "(\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*)([\\+\\-\\*/]?)(\\s*[A-Za-z0-9_\\.]*\\s*)".r + + def replace(replaceStr: String): String = replace(replaceStr, new util.HashMap[String, Any](0)) + + def replace(replaceStr: String, variables: util.Map[String, Any]): String = { + val nameAndType = mutable.Map[String, VariableType]() + var run_date: CustomDateType = null + variables foreach { + case (RUN_DATE, value) if nameAndType.get(RUN_DATE).isEmpty => + val run_date_str = value.asInstanceOf[String] + info(s"replaceStr:${run_date_str}") + if (StringUtils.isNotEmpty(run_date_str)) { + run_date = new CustomDateType(run_date_str, false) + nameAndType(RUN_DATE) = DateType(run_date) + } + case (key, value: String) if !nameAndType.contains(key) && StringUtils.isNotEmpty(value) => + nameAndType(key) = Utils.tryCatch[VariableType](DoubleValue(value.toDouble))(_ => StringType(value)) + case _ => + } + if (!nameAndType.contains(RUN_DATE) || null == run_date) { + val getYesterdayValue = getYesterday(false) + info(s"getYesterday:${getYesterdayValue}") + run_date = new CustomDateType(getYesterdayValue, false) + nameAndType(RUN_DATE) = DateType(new CustomDateType(run_date.toString, false)) + } + initAllDateVars(run_date, nameAndType) + parserVar(replaceStr, nameAndType) + } + + private def initAllDateVars(run_date: CustomDateType, nameAndType: mutable.Map[String, VariableType]): Unit = { + + nameAndType("run_date_std") = DateType(new CustomDateType(run_date.getStdDate)) + nameAndType("run_month_begin") = MonthType(new CustomMonthType(run_date.toString, false)) + nameAndType("run_month_begin_std") = MonthType(new CustomMonthType(run_date.toString)) + nameAndType("run_month_end") = MonthType(new CustomMonthType(run_date.toString, false, true)) + nameAndType("run_month_end_std") = MonthType(new CustomMonthType(run_date.toString, true, true)) + + /* + Variables based on run_today + */ + // if (nameAndType.get(RUN_TODAY).isEmpty || null == run_today) { + // run_today = new CustomDateType(getToday(false), false) + // nameAndType(RUN_TODAY) = DateType(new CustomDateType(run_today.toString, false)) + // } + + val run_today = new CustomDateType(getToday(false, run_date + 1), false) + nameAndType(RUN_TODAY) = DateType(new CustomDateType(run_today.toString, false)) + nameAndType("run_today_std") = DateType(new CustomDateType(run_today.getStdDate)) + nameAndType("run_month_now_begin") = MonthType(new CustomMonthType(new CustomMonthType(run_today.toString, false) - 1, false)) + nameAndType("run_month_now_begin_std") = MonthType(new CustomMonthType(new CustomMonthType(run_today.toString, false) - 1)) + nameAndType("run_month_now_end") = MonthType(new CustomMonthType(new CustomMonthType(run_today.toString, false) - 1, false, true)) + nameAndType("run_month_now_end_std") = MonthType(new CustomMonthType(new CustomMonthType(run_today.toString, false) - 1, true, true)) + + // if (nameAndType.get(RUN_MON).isEmpty || null == run_mon) { + // run_mon = new CustomMonType(getMonthDay(false), false) + // nameAndType(RUN_MON) = MonType(new CustomMonType(run_mon.toString, false)) + // } + + // calculate run_mon base on run_date + val run_mon = new CustomMonType(getMonthDay(false, run_date.getDate), false) + nameAndType("run_mon") = MonType(new CustomMonType(run_mon.toString, false)) + nameAndType("run_mon_std") = MonType(new CustomMonType(run_mon.toString, true, false)) + nameAndType("run_mon_start") = MonType(new CustomMonType(run_mon.toString, false, false)) + nameAndType("run_mon_start_std") = MonType(new CustomMonType(run_mon.toString, true, false)) + nameAndType("run_mon_end") = MonType(new CustomMonType(run_mon.toString, false, true)) + nameAndType("run_mon_end_std") = MonType(new CustomMonType(run_mon.toString, true, true)) + + + nameAndType("run_quarter_begin") = QuarterType(new CustomQuarterType(run_date.toString, false)) + nameAndType("run_quarter_begin_std") = QuarterType(new CustomQuarterType(run_date.toString)) + nameAndType("run_quarter_end") = QuarterType(new CustomQuarterType(run_date.toString, false, true)) + nameAndType("run_quarter_end_std") = QuarterType(new CustomQuarterType(run_date.toString, true, true)) + + nameAndType("run_half_year_begin") = HalfYearType(new CustomHalfYearType(run_date.toString, false)) + nameAndType("run_half_year_begin_std") = HalfYearType(new CustomHalfYearType(run_date.toString)) + nameAndType("run_half_year_end") = HalfYearType(new CustomHalfYearType(run_date.toString, false, true)) + nameAndType("run_half_year_end_std") = HalfYearType(new CustomHalfYearType(run_date.toString, true, true)) + + nameAndType("run_year_begin") = YearType(new CustomYearType(run_date.toString, false)) + nameAndType("run_year_begin_std") = YearType(new CustomYearType(run_date.toString)) + nameAndType("run_year_end") = YearType(new CustomYearType(run_date.toString, false, true)) + nameAndType("run_year_end_std") = YearType(new CustomYearType(run_date.toString, true, true)) + + } + + /** + * Parse and replace the value of the variable + * 1.Get the expression and calculations + * 2.Print user log + * 3.Assemble code + * + * @param replaceStr : replaceStr + * @param nameAndType : variable name and Type + * @return + */ + private def parserVar(replaceStr: String, nameAndType: mutable.Map[String, VariableType]): String = { + val parseCode = new StringBuilder + val codes = codeReg.split(replaceStr) + var i = 0 + codeReg.findAllIn(replaceStr).foreach { str => + i = i + 1 + calReg.findFirstMatchIn(str).foreach { ma => + val name = ma.group(1) + val signal = ma.group(2) + val bValue = ma.group(3) + if (StringUtils.isBlank(name)) { + throw new LinkisCommonErrorException(20041, s"[$str] with empty variable name.") + } + val replacedStr = nameAndType.get(name.trim).map { varType => + if (StringUtils.isNotBlank(signal)) { + if (StringUtils.isBlank(bValue)) { + throw new LinkisCommonErrorException(20042, s"[$str] expression is not right, please check.") + } + varType.calculator(signal.trim, bValue.trim) + } else varType.getValue + }.getOrElse { + warn(s"Use undefined variables or use the set method: [$str](使用了未定义的变量或者使用了set方式:[$str])") + str + } + parseCode ++= codes(i - 1) ++ replacedStr + } + } + if (i == codes.length - 1) { + parseCode ++= codes(i) + } + StringUtils.strip(parseCode.toString) + } + + /** + * Get Yesterday"s date + * + * @param std :2017-11-16 + * @return + */ + private def getYesterday(std: Boolean = true): String = { + val dateFormat = dateFormatLocal.get() + val dateFormat_std = dateFormatStdLocal.get() + val cal: Calendar = Calendar.getInstance() + cal.add(Calendar.DATE, -1) + if (std) { + dateFormat_std.format(cal.getTime) + } else { + dateFormat.format(cal.getTime) + } + } + + // /** + // * Get Month"s date + // * + // * @param std :2017-11-01 + // * @param isEnd :01 or 30,31 + // * @return + // */ + // private[utils] def getMonth(date: Date, std: Boolean = true, isEnd: Boolean = false): String = { + // val dateFormat = dateFormatLocal.get() + // val dateFormat_std = dateFormatStdLocal.get() + // val cal = Calendar.getInstance() + // cal.setTime(date) + // cal.set(Calendar.DATE, 1) + // if (isEnd) { + // cal.roll(Calendar.DATE, -1) + // } + // if (std) { + // dateFormat_std.format(cal.getTime) + // } else { + // dateFormat.format(cal.getTime) + // } + // } + + /** + * Get Today"s date + * + * @param std :2017-11-16 + * @return + */ + def getToday(std: Boolean = true, dateString: String = null): String = { + val dateFormat = dateFormatLocal.get() + val dateFormat_std = dateFormatStdLocal.get() + val cal: Calendar = Calendar.getInstance() + if (dateString != null) { + cal.setTime(dateFormat.parse(dateString)) + } + if (std) { + dateFormat_std.format(cal.getTime) + } else { + dateFormat.format(cal.getTime) + } + } + + + /** + * + * @param std 202106 + * @return + */ + def getMonthDay(std: Boolean = true, date: Date = null): String = { + val dateFormat = dateFormatMonLocal.get() + val dateFormat_std = dateFormatMonStdLocal.get() + if (std) { + dateFormat_std.format(date) + } else { + dateFormat.format(date) + } + } + + /** + * Get Month"s date + * + * @param std :2017-11-01 + * @param isEnd :01 or 30,31 + * @return + */ + def getMonth(std: Boolean = true, isEnd: Boolean = false, date: Date): String = { + val dateFormat = dateFormatLocal.get() + val dateFormat_std = dateFormatStdLocal.get() + val cal: Calendar = Calendar.getInstance() + cal.setTime(date) + cal.set(Calendar.DATE, 1) + if (isEnd) { + cal.roll(Calendar.DATE, -1) + } + if (std) { + dateFormat_std.format(cal.getTime) + } else { + dateFormat.format(cal.getTime) + } + } + + def getMon(std: Boolean = true, isEnd: Boolean = false, date: Date): String = { + val dateFormat = dateFormatMonLocal.get() + val dateFormat_std = dateFormatMonStdLocal.get() + val cal: Calendar = Calendar.getInstance() + cal.setTime(date) + if (isEnd) { + cal.set(Calendar.MONTH, 12) + } + if (std) { + dateFormat_std.format(cal.getTime) + } else { + dateFormat.format(cal.getTime) + } + } + + /** + * get 1st day or last day of a Quarter + * + * @param std + * @param isEnd + * @param date + * @return + */ + def getQuarter(std: Boolean = true, isEnd: Boolean = false, date: Date): String = { + val dateFormat = dateFormatLocal.get() + val dateFormat_std = dateFormatStdLocal.get() + val cal: Calendar = Calendar.getInstance() + cal.setTime(date) + cal.set(Calendar.DATE, 1) + val monthDigit: Int = cal.get(Calendar.MONTH) //get method with MONTH field returns 0-11 + if (0 <= monthDigit && monthDigit <= 2) { + cal.set(Calendar.MONTH, 0) + } else if (3 <= monthDigit && monthDigit <= 5) { + cal.set(Calendar.MONTH, 3) + } else if (6 <= monthDigit && monthDigit <= 8) { + cal.set(Calendar.MONTH, 6) + } else if (9 <= monthDigit && monthDigit <= 11) { + cal.set(Calendar.MONTH, 9) + } + if (isEnd) { + cal.add(Calendar.MONTH, 2) + cal.roll(Calendar.DATE, -1) + } + if (std) { + dateFormat_std.format(cal.getTime) + } else { + dateFormat.format(cal.getTime) + } + } + + /** + * get 1st day or last day of a HalfYear + * + * @param std + * @param isEnd + * @param date + * @return + */ + def getHalfYear(std: Boolean = true, isEnd: Boolean = false, date: Date): String = { + val dateFormat = dateFormatLocal.get() + val dateFormat_std = dateFormatStdLocal.get() + val cal: Calendar = Calendar.getInstance() + cal.setTime(date) + cal.set(Calendar.DATE, 1) + val monthDigit: Int = cal.get(Calendar.MONTH) //get method with MONTH field returns 0-11 + if (0 <= monthDigit && monthDigit <= 5) { + cal.set(Calendar.MONTH, 0) + } else if (6 <= monthDigit && monthDigit <= 11) { + cal.set(Calendar.MONTH, 6) + } + if (isEnd) { + cal.add(Calendar.MONTH, 5) + cal.roll(Calendar.DATE, -1) + } + if (std) { + dateFormat_std.format(cal.getTime) + } else { + dateFormat.format(cal.getTime) + } + } + + /** + * get 1st day or last day of a year + * + * @param std + * @param isEnd + * @param date + * @return + */ + def getYear(std: Boolean = true, isEnd: Boolean = false, date: Date): String = { + val dateFormat = dateFormatLocal.get() + val dateFormat_std = dateFormatStdLocal.get() + val cal: Calendar = Calendar.getInstance() + cal.setTime(date) + cal.set(Calendar.DATE, 1) + cal.set(Calendar.MONTH, 0) // set methods with field MONTH accepts 0-11 + if (isEnd) { + cal.add(Calendar.MONTH, 11) + cal.roll(Calendar.DATE, -1) + } + if (std) { + dateFormat_std.format(cal.getTime) + } else { + dateFormat.format(cal.getTime) + } + } + + + def main(args: Array[String]): Unit = { + val code = "--@set a=1\n--@set b=2\nselect ${a +2},${a + 1},${a},${a },${run_mon},${run_mon_std},${b}" + + val args: java.util.Map[String, Any] = new util.HashMap[String, Any]() + args.put(RUN_DATE, "20181030") + + val str = VariableUtils.replace(code, args) + println(str) + val fixedThreadPool = Executors.newFixedThreadPool(300) + + val task1 = new Runnable { + override def run(): Unit = { + // println("task1 running") + println(Thread.currentThread().getName) + val str = VariableUtils.replace(code, args) + println(str) + // println("task1 complete") + } + } + + + for(a <- 1 to 1000) { + fixedThreadPool.execute(task1) + } + + // println("************code**************") + // var preSQL = "" + // var endSQL = "" + // var sql = "select * from (select * from utf_8_test limit 20) t ;" + // if (sql.contains("limit")) { + // preSQL = sql.substring(0, sql.lastIndexOf("limit")).trim + // endSQL = sql.substring(sql.lastIndexOf("limit")).trim + // } else if (sql.contains("LIMIT")) { + // preSQL = sql.substring(0, sql.lastIndexOf("limit")).trim + // endSQL = sql.substring(sql.lastIndexOf("limit")).trim + // } + // println(preSQL) + // println(endSQL) + /* val yestd = new CustomDateType("2017-11-11",false) + println(yestd)*/ + } + +} + + +trait VariableType { + + def getValue: String + + def calculator(signal: String, bValue: String): String + +} + +case class DateType(value: CustomDateType) extends VariableType { + override def getValue: String = value.toString + + def calculator(signal: String, bValue: String): String = signal match { + case "+" => value + bValue.toInt + case "-" => value - bValue.toInt + case _ => throw new LinkisCommonErrorException(20046, s"Date class is not supported to uss:$signal.") + } +} + + +case class MonthType(value: CustomMonthType) extends VariableType { + override def getValue: String = value.toString + + def calculator(signal: String, bValue: String): String = { + signal match { + case "+" => value + bValue.toInt + case "-" => value - bValue.toInt + case _ => throw new LinkisCommonErrorException(20046, s"Date class is not supported to uss:${signal}") + } + } +} + +case class MonType(value: CustomMonType) extends VariableType { + override def getValue: String = value.toString + + def calculator(signal: String, bValue: String): String = { + signal match { + case "+" => value + bValue.toInt + case "-" => value - bValue.toInt + case _ => throw new LinkisCommonErrorException(20046, s"Date class is not supported to uss:${signal}") + } + } +} + +case class QuarterType(value: CustomQuarterType) extends VariableType { + override def getValue: String = value.toString + + def calculator(signal: String, bValue: String): String = { + signal match { + case "+" => value + bValue.toInt + case "-" => value - bValue.toInt + case _ => throw new LinkisCommonErrorException(20046, s"Date class is not supported to uss:${signal}") + } + } +} + +case class HalfYearType(value: CustomHalfYearType) extends VariableType { + override def getValue: String = value.toString + + def calculator(signal: String, bValue: String): String = { + signal match { + case "+" => value + bValue.toInt + case "-" => value - bValue.toInt + case _ => throw new LinkisCommonErrorException(20046, s"Date class is not supported to uss:${signal}") + } + } +} + +case class YearType(value: CustomYearType) extends VariableType { + override def getValue: String = value.toString + + def calculator(signal: String, bValue: String): String = { + signal match { + case "+" => value + bValue.toInt + case "-" => value - bValue.toInt + case _ => throw new LinkisCommonErrorException(20046, s"Date class is not supported to uss:${signal}") + } + } +} + +case class LongType(value: Long) extends VariableType { + override def getValue: String = value.toString + + def calculator(signal: String, bValue: String): String = signal match { + case "+" => val res = value + bValue.toLong; res.toString + case "-" => val res = value - bValue.toLong; res.toString + case "*" => val res = value * bValue.toLong; res.toString + case "/" => val res = value / bValue.toLong; res.toString + case _ => throw new LinkisCommonErrorException(20047, s"Int class is not supported to uss:$signal.") + } +} + +case class DoubleValue(value: Double) extends VariableType { + override def getValue: String = doubleOrLong(value).toString + + def calculator(signal: String, bValue: String): String = signal match { + case "+" => val res = value + bValue.toDouble; doubleOrLong(res).toString + case "-" => val res = value - bValue.toDouble; doubleOrLong(res).toString + case "*" => val res = value * bValue.toDouble; doubleOrLong(res).toString + case "/" => val res = value / bValue.toDouble; doubleOrLong(res).toString + case _ => throw new LinkisCommonErrorException(20047, s"Double class is not supported to uss:$signal.") + } + + private def doubleOrLong(d: Double): AnyVal = { + if (d.asInstanceOf[Long] == d) d.asInstanceOf[Long] else d + } + +} + +case class FloatType(value: Float) extends VariableType { + override def getValue: String = floatOrLong(value).toString + + def calculator(signal: String, bValue: String): String = signal match { + case "+" => val res = value + bValue.toFloat; floatOrLong(res).toString + case "-" => val res = value - bValue.toFloat; floatOrLong(res).toString + case "*" => val res = value * bValue.toFloat; floatOrLong(res).toString + case "/" => val res = value / bValue.toLong; floatOrLong(res).toString + case _ => throw new LinkisCommonErrorException(20048, s"Float class is not supported to use:$signal.") + } + + private def floatOrLong(f: Float): AnyVal = { + if (f.asInstanceOf[Long] == f) f.asInstanceOf[Long] else f + } + +} + +case class StringType(value: String) extends VariableType { + override def getValue: String = value.toString + + def calculator(signal: String, bValue: String): String = signal match { + case "+" => value + bValue + case _ => throw new LinkisCommonErrorException(20049, s"String class is not supported to uss:$signal.") + } +} + +import VariableUtils._ + +class CustomDateType(date: String, std: Boolean = true) { + + + + def -(days: Int): String = { + if (std) { + dateFormatStdLocal.get().format(DateUtils.addDays( dateFormatStdLocal.get().parse(date), -days)) + } else { + dateFormatLocal.get().format(DateUtils.addDays(dateFormatLocal.get().parse(date), -days)) + } + } + + def +(days: Int): String = { + if (std) { + dateFormatStdLocal.get().format(DateUtils.addDays( dateFormatStdLocal.get().parse(date), days)) + } else { + dateFormatLocal.get().format(DateUtils.addDays(dateFormatLocal.get().parse(date), days)) + } + } + + def getDate: Date = { + if (std) { + dateFormatStdLocal.get().parse(date) + } else { + dateFormatLocal.get().parse(date) + } + } + + def getStdDate: String = { + if (std) { + dateFormatStdLocal.get().format( dateFormatStdLocal.get().parse(date)) + } else { + dateFormatStdLocal.get().format(dateFormatLocal.get().parse(date)) + } + } + + override def toString: String = { + if (std) { + dateFormatStdLocal.get().format( dateFormatStdLocal.get().parse(date)) + } else { + dateFormatLocal.get().format(dateFormatLocal.get().parse(date)) + } + } +} + + +class CustomMonthType(date: String, std: Boolean = true, isEnd: Boolean = false) { + + def -(months: Int): String = { + if (std) { + VariableUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormatLocal.get().parse(date), -months)) + } else { + VariableUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormatLocal.get().parse(date), -months)) + } + } + + def +(months: Int): String = { + if (std) { + VariableUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormatLocal.get().parse(date), months)) + } else { + VariableUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormatLocal.get().parse(date), months)) + } + } + + override def toString: String = { + if (std) { + VariableUtils.getMonth(std, isEnd, dateFormatLocal.get().parse(date)) + } else { + val v = dateFormatLocal.get().parse(date) + VariableUtils.getMonth(std, isEnd, v) + } + } + +} + +class CustomMonType(date: String, std: Boolean = true, isEnd: Boolean = false) { + + def -(months: Int): String = { + if (std) { + VariableUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormatMonLocal.get().parse(date), -months)) + } else { + VariableUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormatMonLocal.get().parse(date), -months)) + } + } + + def +(months: Int): String = { + if (std) { + VariableUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormatMonLocal.get().parse(date), months)) + } else { + VariableUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormatMonLocal.get().parse(date), months)) + } + } + + override def toString: String = { + if (std) { + VariableUtils.getMon(std, isEnd, dateFormatMonLocal.get().parse(date)) + } else { + val v = dateFormatMonLocal.get().parse(date) + VariableUtils.getMon(std, isEnd, v) + } + } + +} + +/* + Given a Date, convert into Quarter + */ +class CustomQuarterType(date: String, std: Boolean = true, isEnd: Boolean = false) { + + + def getCurrentQuarter(date: String) = { + dateFormatLocal.get().parse(VariableUtils.getQuarter(false, false, dateFormatLocal.get().parse(date))) + } + + def -(quarters: Int): String = { + VariableUtils.getQuarter(std, isEnd, DateUtils.addMonths(getCurrentQuarter(date), -quarters * 3)) + } + + def +(quarters: Int): String = { + VariableUtils.getQuarter(std, isEnd, DateUtils.addMonths(getCurrentQuarter(date), quarters * 3)) + } + + override def toString: String = { + if (std) { + VariableUtils.getQuarter(std, isEnd, dateFormatLocal.get().parse(date)) + } else { + val v = dateFormatLocal.get().parse(date) + VariableUtils.getQuarter(std, isEnd, v) + } + } + +} + +/* + Given a Date, convert into HalfYear + */ +class CustomHalfYearType(date: String, std: Boolean = true, isEnd: Boolean = false) { + + + def getCurrentHalfYear(date: String) = { + dateFormatLocal.get().parse(VariableUtils.getHalfYear(false, false, dateFormatLocal.get().parse(date))) + } + + def -(halfYears: Int): String = { + VariableUtils.getHalfYear(std, isEnd, DateUtils.addMonths(getCurrentHalfYear(date), -halfYears * 6)) + } + + def +(halfYears: Int): String = { + VariableUtils.getHalfYear(std, isEnd, DateUtils.addMonths(getCurrentHalfYear(date), halfYears * 6)) + } + + override def toString: String = { + if (std) { + VariableUtils.getHalfYear(std, isEnd, dateFormatLocal.get().parse(date)) + } else { + val v = dateFormatLocal.get().parse(date) + VariableUtils.getHalfYear(std, isEnd, v) + } + } + +} + +/* + Given a Date convert into Year + */ +class CustomYearType(date: String, std: Boolean = true, isEnd: Boolean = false) { + + def -(years: Int): String = { + VariableUtils.getYear(std, isEnd, DateUtils.addYears(dateFormatLocal.get().parse(date), -years)) + } + + def +(years: Int): String = { + VariableUtils.getYear(std, isEnd, DateUtils.addYears(dateFormatLocal.get().parse(date), years)) + } + + override def toString: String = { + if (std) { + VariableUtils.getYear(std, isEnd, dateFormatLocal.get().parse(date)) + } else { + val v = dateFormatLocal.get().parse(date) + VariableUtils.getYear(std, isEnd, v) + } + } + +} diff --git a/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/conf/DSSConfiguration.scala b/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/conf/DSSConfiguration.scala new file mode 100644 index 000000000..f1a788709 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/conf/DSSConfiguration.scala @@ -0,0 +1,9 @@ +package org.apache.linkis.common.conf + +import java.util.Properties + +object DSSConfiguration { + + def getAllProperties: Properties = BDPConfiguration.properties + +} diff --git a/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/utils/DefaultRetryHandler.scala b/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/utils/DefaultRetryHandler.scala new file mode 100644 index 000000000..7dc6a46fa --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/utils/DefaultRetryHandler.scala @@ -0,0 +1,9 @@ +package org.apache.linkis.common.utils + +/** + * + * @date 2022-03-18 + * @author enjoyyin + * @since 1.1.0 + */ +class DefaultRetryHandler extends RetryHandler diff --git a/dss-commons/dss-contextservice/pom.xml b/dss-commons/dss-contextservice/pom.xml new file mode 100644 index 000000000..4d9c1d10f --- /dev/null +++ b/dss-commons/dss-contextservice/pom.xml @@ -0,0 +1,84 @@ + + + + 4.0.0 + + dss-commons + com.webank.wedatasphere.dss + 1.1.1 + + dss-contextservice + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + + org.apache.linkis + linkis-cs-client + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-scheduler-appconn + ${dss.version} + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + \ No newline at end of file diff --git a/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/ContextService.java b/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/ContextService.java new file mode 100644 index 000000000..b4fafea3c --- /dev/null +++ b/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/ContextService.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.contextservice.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + +public interface ContextService { + + /** + * 创建ContextID,返回ContextID序列化后的字符串 + * @param flow + * @param user + * @param version + * @return + */ + String createContextID(String workspace, String projectName, String flow, String version, String user); + + /** + * 检查和创建ContextID,返回创建ID后的jsonFlow + * 检查ContextID信息:如果jsonFlow不包含ContextID信息,则创建新的; + * 如果已经有ContextID,判断传入flowVersion是否相同,不同则创建新的ContextID,并返回新创建的ContextID序列化内容;相同则不创建新ID; + * 新创建的ContextID会更新到flowJson里 + * @param jsonFlow + * @param flowVersion + * @param workspace + * @param project + * @param flow + * @param user + * @param fullCheck - true 检查所有参数; false 只检查jsonFlow中ContextID是否存在,存在则不再比较ContextID中version参数等是否相同,且不创建 + * @return + */ + String checkAndCreateContextID(String jsonFlow, String flowVersion, String workspace, String project, String flow, String user, boolean fullCheck); + + /** + * 解析jsonFlow、DWSProject内容,存储到CS里 + * 解析DWSProject、jsonFlow里面的资源、变量、节点依赖等信息,存储到CS服务 + * @param jsonFlow + * @param parentFlowID + */ + void checkAndSaveContext(String jsonFlow, String parentFlowID) throws DSSErrorException; + + /** + * 检查更新ContextID信息,解析传入的jsonFlow、DWSProject内容,存储到CS里 + * @param jsonFlow 必需 + * @param parentFlowId 必需 + * @param workspace + * @param flowVersion + * @param user + * @throws DSSErrorException + */ + String checkAndInitContext(String jsonFlow, String parentFlowId, String workspace, String projectName, String flowName, String flowVersion, String user) throws DSSErrorException; + + /** + * 解析SchedulerFlow信息,并更新ContextID和存储Context信息,调用checkAndInitContext信息 + * @param schedulerFlow + * @return + * @throws DSSErrorException + */ +// String checkAndInitContext(SchedulerFlow schedulerFlow) throws DSSErrorException; +} diff --git a/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/impl/ContextServiceImpl.java b/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/impl/ContextServiceImpl.java new file mode 100644 index 000000000..bd9a7d69f --- /dev/null +++ b/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/impl/ContextServiceImpl.java @@ -0,0 +1,412 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.contextservice.service.impl; + +import com.google.gson.*; +import com.webank.wedatasphere.dss.common.conf.DSSCommonConf; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.exception.ErrorCode; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.cs.client.ContextClient; +import org.apache.linkis.cs.client.builder.ContextClientFactory; +import org.apache.linkis.cs.client.service.CSWorkService; +import org.apache.linkis.cs.client.service.CSWorkServiceImpl; +import org.apache.linkis.cs.client.utils.SerializeHelper; +import org.apache.linkis.cs.common.entity.enumeration.ContextScope; +import org.apache.linkis.cs.common.entity.enumeration.ContextType; +import org.apache.linkis.cs.common.entity.enumeration.WorkType; +import org.apache.linkis.cs.common.entity.object.CSFlowInfos; +import org.apache.linkis.cs.common.entity.object.LinkisVariable; +import org.apache.linkis.cs.common.entity.resource.LinkisBMLResource; +import org.apache.linkis.cs.common.entity.source.*; +import org.apache.linkis.cs.common.utils.CSCommonUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public class ContextServiceImpl implements ContextService { + + private static final Logger logger = LoggerFactory.getLogger(ContextServiceImpl.class); + private static ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + private static ContextService contextService = null; + + private ContextServiceImpl() {} + + public static ContextService getInstance() { + if (null == contextService) { + synchronized (ContextServiceImpl.class) { + if (null == contextService) { + contextService = new ContextServiceImpl(); + } + } + } + return contextService; + } + + @Override + public String createContextID(String workspace, String project, String flow, String version, String user) { + LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); + contextID.setWorkSpace(workspace); + contextID.setProject(project); + contextID.setFlow(flow); + contextID.setUser(user); + contextID.setVersion(version); + contextID.setEnv(DSSCommonConf.DSS_IO_ENV.getValue()); + try { + contextClient.createContext(contextID); + return SerializeHelper.serializeContextID(contextID); + } catch (Exception e) { + logger.error("createContextID error. workspace : {}, project : {}, flow : {}, version : {}, user : {}", workspace, project, flow, version, user, e); + throw new DSSRuntimeException(50032, "Try to ask Linkis for creating a new contextId failed(向Linkis请求创建一个ContextID失败)! Linkis error msg: " + + ExceptionUtils.getRootCauseMessage(e), e); + } + } + + @Override + public String checkAndCreateContextID(String jsonFlow, String flowVersion, String workspace, String project, String flow, String user, boolean fullCheck) { + JsonObject flowObject = null; + try { + flowObject = new Gson().fromJson(jsonFlow, JsonObject.class); + if (!flowObject.has(CSCommonUtils.CONTEXT_ID_STR) || + StringUtils.isBlank(flowObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString())|| + !flowObject.get(CSCommonUtils.CONTEXT_ID_STR).isJsonPrimitive()) { + String contextID = createContextID(workspace, + project, + flow, flowVersion, user); + flowObject.addProperty(CSCommonUtils.CONTEXT_ID_STR, contextID); + } else { + boolean updateContextId = false; + LinkisHAWorkFlowContextID contextID = null; + try { + Object contextIDObj = SerializeHelper.deserializeContextID(flowObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString()); + contextID = (LinkisHAWorkFlowContextID)contextIDObj; + if (fullCheck) { + if (!DSSCommonConf.DSS_IO_ENV.getValue().equalsIgnoreCase(contextID.getEnv())) { + updateContextId = true; + } else if (StringUtils.isBlank(contextID.getProject()) + || StringUtils.isBlank(contextID.getFlow()) + || StringUtils.isBlank(contextID.getVersion())) { + updateContextId = false; + } else if ((null != contextID.getWorkSpace() && !contextID.getWorkSpace().equalsIgnoreCase(workspace)) + || !contextID.getProject().equalsIgnoreCase(project) + || !contextID.getFlow().equalsIgnoreCase(flow) + || !contextID.getVersion().equalsIgnoreCase(flowVersion)) { + updateContextId = true; + } else { + updateContextId = false; + } + } + } catch (ErrorException e0) { + logger.error("Invalid contextID : {}, please contact with administrator", flowObject.get(CSCommonUtils.CONTEXT_ID_STR)); + } + if (updateContextId) { + String newContextID = createContextID(workspace, + project, + flow, flowVersion, user); + flowObject.addProperty(CSCommonUtils.CONTEXT_ID_STR, newContextID); + logger.info("UpdateContextId true, old contextID : {}, new contextID : {}", CSCommonUtils.gson.toJson(contextID), CSCommonUtils.gson.toJson(newContextID)); + } + } + } catch (Exception e) { + logger.error("Invalid json : {}", jsonFlow, e); + } + if (null != flowObject) { + return flowObject.toString(); + } else { + return jsonFlow; + } + } + + @Override + public void checkAndSaveContext(String jsonFlow,String parentFlowID) throws DSSErrorException{ + logger.info("jsonFlow => \n" + jsonFlow); + try { + JsonObject flowObject = new Gson().fromJson(jsonFlow, JsonObject.class); + if (!flowObject.has(CSCommonUtils.CONTEXT_ID_STR) || !flowObject.get(CSCommonUtils.CONTEXT_ID_STR).isJsonPrimitive()) { + logger.error("Did not have invalid contextID, save context failed."); + return; + } else { + String contextIDStr = flowObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString(); + // ①reset原有key 这里只清理 + CSWorkService csWorkService = CSWorkServiceImpl.getInstance(); + List workTypes = new ArrayList<>(); + workTypes.add(WorkType.WORKSPACE); + workTypes.add(WorkType.PROJECT); + workTypes.add(WorkType.FLOW); + csWorkService.initContextServiceInfo(contextIDStr, workTypes); + + // 保存flow的资源 + if (flowObject.has(DSSCommonUtils.FLOW_RESOURCE_NAME)) { + JsonArray flowRes = flowObject.get(DSSCommonUtils.FLOW_RESOURCE_NAME).getAsJsonArray(); + saveContextResource(contextIDStr, flowRes, contextClient, CSCommonUtils.FLOW_RESOURCE_PREFIX, null); + } + if (flowObject.has(DSSCommonUtils.FLOW_PROP_NAME)) { + JsonElement flowProp = flowObject.get(DSSCommonUtils.FLOW_PROP_NAME); + saveContextVariable(contextIDStr, flowProp, contextClient, CSCommonUtils.FLOW_VARIABLE_PREFIX, null); + } + // todo udf + + // 保存节点的资源 + if (flowObject.has(DSSCommonUtils.FLOW_NODE_NAME)) { + JsonArray nodes = flowObject.get(DSSCommonUtils.FLOW_NODE_NAME).getAsJsonArray(); + for (JsonElement node : nodes) { + JsonObject json = node.getAsJsonObject(); + String nodeName =json.get(DSSCommonUtils.NODE_NAME_NAME).getAsString(); + initContextNodeVarInfo(contextIDStr,nodeName,contextClient); + if (json.has(DSSCommonUtils.NODE_RESOURCE_NAME)) { + JsonArray nodeRes = json.get(DSSCommonUtils.NODE_RESOURCE_NAME).getAsJsonArray(); + saveContextResource(contextIDStr, nodeRes, contextClient, + CSCommonUtils.NODE_PREFIX, json.get(DSSCommonUtils.NODE_NAME_NAME).getAsString()); + } + if (json.has(DSSCommonUtils.NODE_PROP_NAME)) { + JsonObject nodePropObj = json.get(DSSCommonUtils.NODE_PROP_NAME).getAsJsonObject(); + } + } + } + // 保存info信息 + saveFlowInfo(contextIDStr, parentFlowID, jsonFlow); + } + } catch (Exception e) { + logger.error("CheckAndSaveContext error. jsonFlow : {}, parentFlowId : {}, e : ", jsonFlow, parentFlowID, e); + throw new DSSErrorException(ErrorCode.INVALID_PARAMS, "CheckAndSaveContext error : " + e.getMessage()); + } + } + + private void initContextNodeVarInfo(String contextIDStr,String nodeName,ContextClient contextClient){ + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + contextClient.removeAllValueByKeyPrefixAndContextType(contextID, ContextType.Variable, CSCommonUtils.NODE_PREFIX + nodeName); + } catch (ErrorException e) { + logger.error("CheckAndSaveContext error. ContextID : {}, nodeName : {}, e : ", contextIDStr,nodeName, e); + } + } + + @Override + public String checkAndInitContext(String jsonFlow, String parentFlowId, String workspace, String projectName, String flowName, String flowVersion, String user) throws DSSErrorException { + if (StringUtils.isBlank(jsonFlow) ) { + logger.error("Invalid jsonFlow : {} or parentFlowId : {}.", jsonFlow, parentFlowId); + throw new DSSErrorException(ErrorCode.INVALID_PARAMS, "Invalid jsonFlow : " + jsonFlow + ", or parentFlowId : " + parentFlowId + "."); + } + jsonFlow = checkAndCreateContextID(jsonFlow, flowVersion, workspace, projectName, flowName, user, true); + checkAndSaveContext(jsonFlow,parentFlowId); + JsonObject flowObject = new Gson().fromJson(jsonFlow, JsonObject.class); + return flowObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString(); + } + + + + private void saveContextVariable(String contextIDStr, JsonElement variables, ContextClient contextClient, String variablePrefix, String nodeName) { + try { + if (variables.isJsonArray()) { + JsonArray flowProp = variables.getAsJsonArray(); + for (JsonElement prop : flowProp) { + Set> entrySet = prop.getAsJsonObject().entrySet(); + // assign entry num is 1 + if (null != entrySet) { + for (Map.Entry entry : entrySet) { + if (!entry.getValue().isJsonPrimitive()) { + continue; + } + saveContextVariableKeyValue(contextIDStr, contextClient, variablePrefix, entry, nodeName); + } + } + + } + } else if (variables.isJsonObject()) { + JsonObject variableJson = variables.getAsJsonObject(); + for (Map.Entry entry : variableJson.entrySet()) { + if (!entry.getValue().isJsonPrimitive()) { + continue; + } + saveContextVariableKeyValue(contextIDStr, contextClient, variablePrefix, entry, nodeName); + } + } else { + logger.error("Invalid JsonElement variables : {}", variables.toString()); + } + + } catch (ErrorException e) { + logger.error("SaveContextVariable failed. contextIDStr : {}, variables : {}, variablePrefix : {}, e : ", + contextIDStr, variables.toString(), variablePrefix, e); + } + + } + + private void saveContextVariableKeyValue(String contextIDStr, ContextClient contextClient, String uniKeyPrefix, Map.Entry entry, String nodeName) throws ErrorException { + String contextKeyPrefix = null; + switch (uniKeyPrefix) { + case CSCommonUtils.WORKSPACE_VARIABLE_PREFIX: + case CSCommonUtils.PROJECT_VARIABLE_PREFIX: + case CSCommonUtils.FLOW_VARIABLE_PREFIX: + contextKeyPrefix = uniKeyPrefix; + break; + case CSCommonUtils.NODE_PREFIX: + contextKeyPrefix = uniKeyPrefix + nodeName + "." + CSCommonUtils.VARIABLE_PREFIX; + break; + default: + logger.error("Invalid contextKeyPrefix : {}", uniKeyPrefix); + return; + } + LinkisVariable linkisVariable = new LinkisVariable(); + linkisVariable.setKey(entry.getKey()); + linkisVariable.setValue(entry.getValue().getAsString()); + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey(contextKeyPrefix + linkisVariable.getKey()); + contextKey.setContextType(ContextType.Variable); + contextKey.setContextScope(ContextScope.PUBLIC); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(linkisVariable); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + contextClient.setContextKeyValue(SerializeHelper.deserializeContextID(contextIDStr), contextKeyValue); + } + + private void saveContextResource(String contextIDStr, JsonArray flowRes, ContextClient contextClient, String uniKeyPrefix, String nodeName) throws ErrorException { + String contextKeyPrefix = null; + switch (uniKeyPrefix) { + case CSCommonUtils.WORKSPACE_RESOURCE_PREFIX: + case CSCommonUtils.PROJECT_RESOURCE_PREFIX: + case CSCommonUtils.FLOW_RESOURCE_PREFIX: + contextKeyPrefix = uniKeyPrefix; + break; + case CSCommonUtils.NODE_PREFIX: + contextKeyPrefix = uniKeyPrefix + nodeName + "." + CSCommonUtils.RESOURCE_PREFIX; + break; + default: + logger.error("Invalid contextKeyPrefix : {}", uniKeyPrefix); + return; + } + for (JsonElement res : flowRes) { + LinkisBMLResource bmlResource = new LinkisBMLResource(); + JsonObject json = res.getAsJsonObject(); + if (!json.has("fileName") || !json.has("resourceId") || !json.has("version")) { + logger.warn("Invalid resource: res : {}, contextidStr : {}, all res : {} ", CSCommonUtils.gson.toJson(json), contextIDStr, CSCommonUtils.gson.toJson(flowRes)); + continue; + } + if (json.get("fileName") instanceof JsonNull || json.get("resourceId") instanceof JsonNull || json.get("version") instanceof JsonNull) { + logger.warn("Invalid resource: res : {}, contextidStr : {}, all res : {} ", CSCommonUtils.gson.toJson(json), contextIDStr, CSCommonUtils.gson.toJson(flowRes)); + continue; + } + bmlResource.setDownloadedFileName(json.get("fileName").getAsString()); + bmlResource.setResourceId(json.get("resourceId").getAsString()); + bmlResource.setVersion(json.get("version").getAsString()); + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey(contextKeyPrefix + bmlResource.getDownloadedFileName()); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(bmlResource); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + contextClient.setContextKeyValue(SerializeHelper.deserializeContextID(contextIDStr), contextKeyValue); + //// todo test + logger.info("Debug: saved contextKeyValue : {}", CSCommonUtils.gson.toJson(contextKeyValue)); + } + } + + private void saveContextResource(String contextIDStr, List resourceList, ContextClient contextClient, String uniKeyPrefix) throws ErrorException { + + String contextKeyPrefix = null; + switch (uniKeyPrefix) { + case CSCommonUtils.WORKSPACE_RESOURCE_PREFIX: + case CSCommonUtils.PROJECT_RESOURCE_PREFIX: + case CSCommonUtils.FLOW_RESOURCE_PREFIX: + contextKeyPrefix = uniKeyPrefix; + break; + default: + logger.error("Invalid contextKeyPrefix : {}", uniKeyPrefix); + return; + } + for (Resource res : resourceList) { + LinkisBMLResource bmlResource = new LinkisBMLResource(); + bmlResource.setDownloadedFileName(res.getFileName()); + bmlResource.setResourceId(res.getResourceId()); + bmlResource.setVersion(res.getVersion()); + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey(contextKeyPrefix + bmlResource.getDownloadedFileName()); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(bmlResource); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + contextClient.setContextKeyValue(SerializeHelper.deserializeContextID(contextIDStr), contextKeyValue); + //// todo test + logger.info("Debug: saved contextKeyValue : {}", CSCommonUtils.gson.toJson(contextKeyValue)); + } + } + + + /** + * 保存节点间关系信息 + * @param contextIDStr + * @param parentFlowID + * @param parentFlowID + */ + private void saveFlowInfo(String contextIDStr, String parentFlowID, String flowJson) { + if (StringUtils.isBlank(contextIDStr)) { + return; + } + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + JsonObject flowJsonObject = new Gson().fromJson(flowJson, JsonObject.class); + CSFlowInfos flowInfos = new CSFlowInfos(); + Map flowInfoMap = new HashMap<>(); + // 保存边信息 + JsonArray flowEdges = new JsonArray(); + if (flowJsonObject.has(DSSCommonUtils.FLOW_EDGES_NAME)) { + flowEdges = flowJsonObject.getAsJsonArray(DSSCommonUtils.FLOW_EDGES_NAME); + } + flowInfoMap.put(DSSCommonUtils.FLOW_EDGES_NAME, flowEdges); + // 保存父节点 可以为空 + flowInfoMap.put(DSSCommonUtils.FLOW_PARENT_NAME, parentFlowID); + //保存节点名 + JsonObject idNodeNameJson = new JsonObject(); + if (flowJsonObject.has(DSSCommonUtils.FLOW_NODE_NAME)) { + JsonArray nodes = flowJsonObject.getAsJsonArray(DSSCommonUtils.FLOW_NODE_NAME); + for (JsonElement element : nodes) { + JsonObject jsonObject = element.getAsJsonObject(); + String id = jsonObject.get(DSSCommonUtils.NODE_ID_NAME).getAsString(); + String name = jsonObject.get(DSSCommonUtils.NODE_NAME_NAME).getAsString(); + idNodeNameJson.addProperty(id, name); + } + } + flowInfoMap.put(CSCommonUtils.ID_NODE_NAME, idNodeNameJson); + flowInfos.setInfos(flowInfoMap); + ContextKey contextKey = new CommonContextKey(); + contextKey.setContextType(ContextType.OBJECT); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setKey(CSCommonUtils.FLOW_INFOS); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(flowInfos); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + try { + contextClient.setContextKeyValue(SerializeHelper.deserializeContextID(contextIDStr), contextKeyValue); + //// todo test + logger.info("ContextID : {}, \nContextKey : {}, \nContextValue : {}", + CSCommonUtils.gson.toJson(SerializeHelper.deserializeContextID(contextIDStr)), + CSCommonUtils.gson.toJson(contextKey), + CSCommonUtils.gson.toJson(contextValue)); + ContextValue contextValue1 = contextClient.getContextValue(SerializeHelper.deserializeContextID(contextIDStr), contextKey); + logger.info(CSCommonUtils.gson.toJson(contextValue1)); + } catch (ErrorException e) { + logger.error("Set ContextKeyValue error. contextIDStr "); + } + } +} \ No newline at end of file diff --git a/dss-commons/dss-sender-service/pom.xml b/dss-commons/dss-sender-service/pom.xml new file mode 100644 index 000000000..6be866d15 --- /dev/null +++ b/dss-commons/dss-sender-service/pom.xml @@ -0,0 +1,57 @@ + + + + + + dss-commons + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-sender-service + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + + org.apache.linkis + linkis-rpc + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + provided + + + + + \ No newline at end of file diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java new file mode 100644 index 000000000..212d631ec --- /dev/null +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.sender.service; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import org.apache.linkis.rpc.Sender; +import java.util.List; + +public interface DSSSenderService { + + Sender getOrcSender(); + + Sender getOrcSender(List dssLabels); + + Sender getScheduleOrcSender(); + + Sender getWorkflowSender(List dssLabels); + + Sender getWorkflowSender(); + + Sender getSchedulerWorkflowSender(); + + Sender getProjectServerSender(); + +} + diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderServiceFactory.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderServiceFactory.java new file mode 100644 index 000000000..d797ddd72 --- /dev/null +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderServiceFactory.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.sender.service; + +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.sender.service.impl.DSSSenderServiceImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DSSSenderServiceFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSSenderServiceFactory.class); + private static final DSSSenderService service; + + static { + service = ClassUtils.getInstanceOrDefault(DSSSenderService.class, new DSSSenderServiceImpl()); + LOGGER.info("Use {} to instance a available DSSSenderService.", service.getClass().getName()); + } + + + public static DSSSenderService getOrCreateServiceInstance() { + return service; + } + + +} diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java new file mode 100644 index 000000000..5a551c5f7 --- /dev/null +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.sender.service.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class DSSSenderServiceConf { + public static final CommonVars ORCHESTRATOR_SERVER_DEV_NAME = + CommonVars.apply("wds.dss.orc.server.dev.name", "DSS-Framework-Orchestrator-Server-Dev"); + + public static final CommonVars DSS_WORKFLOW_APPLICATION_NAME_DEV = + CommonVars.apply("wds.dss.workflow.name.dev", "dss-workflow-server-dev"); + + public static final CommonVars PROJECT_SERVER_NAME = + CommonVars.apply("wds.dss.project.sever.name", "dss-framework-project-server"); + +} diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java new file mode 100644 index 000000000..ff403f86c --- /dev/null +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.sender.service.impl; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.sender.service.DSSSenderService; +import com.webank.wedatasphere.dss.sender.service.conf.DSSSenderServiceConf; +import org.apache.linkis.rpc.Sender; +import java.util.List; + +public class DSSSenderServiceImpl implements DSSSenderService { + + private final Sender orcSender = Sender.getSender(DSSSenderServiceConf.ORCHESTRATOR_SERVER_DEV_NAME.getValue()); + + private final Sender workflowSender = Sender.getSender(DSSSenderServiceConf.DSS_WORKFLOW_APPLICATION_NAME_DEV.getValue()); + + private final Sender projectSender = Sender.getSender(DSSSenderServiceConf.PROJECT_SERVER_NAME.getValue()); + @Override + public Sender getOrcSender() { + return orcSender; + } + + @Override + public Sender getOrcSender(List dssLabels) { + return orcSender; + } + + @Override + public Sender getScheduleOrcSender() { + return orcSender; + } + + @Override + public Sender getWorkflowSender(List dssLabels) { + return workflowSender; + } + + @Override + public Sender getWorkflowSender() { + return workflowSender; + } + + @Override + public Sender getSchedulerWorkflowSender() { + return workflowSender; + } + + @Override + public Sender getProjectServerSender() { + return projectSender; + } + +} diff --git a/dss-commons/pom.xml b/dss-commons/pom.xml new file mode 100644 index 000000000..1c4e33593 --- /dev/null +++ b/dss-commons/pom.xml @@ -0,0 +1,37 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-commons + pom + + + dss-common + dss-contextservice + dss-sender-service + + + \ No newline at end of file diff --git a/dss-framework/dss-appconn-framework/pom.xml b/dss-framework/dss-appconn-framework/pom.xml new file mode 100644 index 000000000..9cea7ebf9 --- /dev/null +++ b/dss-framework/dss-appconn-framework/pom.xml @@ -0,0 +1,127 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + + 4.0.0 + + dss-appconn-framework + + + + junit + junit + 4.12 + test + + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + + com.webank.wedatasphere.dss + dss-appconn-manager-core + ${dss.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + linkis-common + org.apache.linkis + + + + + org.apache.linkis + linkis-bml-client + ${linkis.version} + + + commons-beanutils + commons-beanutils + + + linkis-common + org.apache.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + jakarta.annotation + jakarta.annotation-api + 1.3.5 + provided + + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/SpringAppConnManager.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/SpringAppConnManager.java new file mode 100644 index 000000000..952c87e71 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/SpringAppConnManager.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn; + +import com.webank.wedatasphere.dss.appconn.manager.impl.AbstractAppConnManager; +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnInfoService; +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnResourceService; +import org.apache.linkis.DataWorkCloudApplication; + + +public class SpringAppConnManager extends AbstractAppConnManager { + + @Override + protected AppConnInfoService createAppConnInfoService() { + return DataWorkCloudApplication.getApplicationContext().getBean(AppConnInfoService.class); + } + + @Override + protected AppConnResourceService createAppConnResourceService() { + return DataWorkCloudApplication.getApplicationContext().getBean(AppConnResourceService.class); + } +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/conf/AppConnConf.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/conf/AppConnConf.java new file mode 100644 index 000000000..c0c1d17ef --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/conf/AppConnConf.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.framework.appconn.conf; + +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.CommonVars; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class AppConnConf { + + public static final CommonVars IS_APPCONN_MANAGER = CommonVars.apply("wds.dss.appconn.framework.ismanager", true); + + public static final CommonVars PROJECT_QUALITY_CHECKER_IGNORE_LIST = CommonVars.apply("wds.dss.appconn.checker.project.ignore.list", ""); + + public static final CommonVars DEVELOPMENT_QUALITY_CHECKER_IGNORE_LIST = CommonVars.apply("wds.dss.appconn.checker.development.ignore.list", ""); + + public static final List DISABLED_APP_CONNS = getDisabledAppConns(); + + private static List getDisabledAppConns() { + String disabledAppConns = CommonVars.apply("wds.dss.appconn.disabled", "").getValue(); + if(StringUtils.isBlank(disabledAppConns)) { + return Collections.emptyList(); + } else { + return Arrays.asList(disabledAppConns.split(",")); + } + } + +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/AppConnMapper.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/AppConnMapper.java new file mode 100644 index 000000000..7eaa59040 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/AppConnMapper.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.dao; + + +import com.webank.wedatasphere.dss.framework.appconn.entity.AppConnBean; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +@Mapper +public interface AppConnMapper { + + /** + * get all appconnbeans + * */ + List getAllAppConnBeans(); + + /** + * get all appconns' name + * */ + List getAllAppConnsName(); + + /** + * get appconnbeans by name + * */ + AppConnBean getAppConnBeanByName(@Param("appConnName") String appConnName); + + /** + * get appconn by id + * */ + AppConnBean getAppConnBeanById(@Param("appConnId") Long appConnId); + + void updateResourceByName(AppConnBean appConnBean); + +} \ No newline at end of file diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/AppInstanceMapper.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/AppInstanceMapper.java new file mode 100644 index 000000000..701506701 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/AppInstanceMapper.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.dao; + + +import com.webank.wedatasphere.dss.framework.appconn.entity.AppInstanceBean; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +@Mapper +public interface AppInstanceMapper { + + /** + * get instance by appconnid + * */ + List getAppInstancesByAppConnId(@Param("appConnId") Long appConnId); + + /** + * get instance by appconnid and label + * */ + List getAppInstance(@Param("appConnId") Long appConnId, + @Param("label") String label); +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/impl/appConnMapper.xml b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/impl/appConnMapper.xml new file mode 100644 index 000000000..2473ceae0 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/impl/appConnMapper.xml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + `appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, + `reference`, `class_name`, `resource` + + + + `id`, `appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, + `reference`, `class_name`, `resource` + + + + + + + + + + + + update `dss_appconn` + + resource=#{resource}, + + where appconn_name=#{appConnName} + + \ No newline at end of file diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/impl/appInstanceMapper.xml b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/impl/appInstanceMapper.xml new file mode 100644 index 000000000..8b05e8256 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/dao/impl/appInstanceMapper.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + `id`, `appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri` + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppConnBean.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppConnBean.java new file mode 100644 index 000000000..6fd4f6a48 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppConnBean.java @@ -0,0 +1,147 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.entity; + +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import com.webank.wedatasphere.dss.common.entity.Resource; + +import java.io.Serializable; +import java.util.List; + + + +public class AppConnBean implements AppConnInfo, Serializable { + + private static final long serialVersionUID=1L; + private Long id; + private String appConnName; + private String isUserNeedInit; + private String level; + private Boolean ifIframe; + private Boolean isExternal; + + // todo:目前通过这两个字段在classpath下加载类。 + // 未来这两个字段的作用是在bml和默认应用的。 + private String reference; + private String resource; + private Resource resourceObj; + private String className; + + private List appInstanceBeans; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Override + public String getAppConnName() { + return appConnName; + } + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + public String getIsUserNeedInit() { + return isUserNeedInit; + } + + public void setIsUserNeedInit(String isUserNeedInit) { + this.isUserNeedInit = isUserNeedInit; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + public Boolean getIfIframe() { + return ifIframe; + } + + public void setIfIframe(Boolean ifIframe) { + this.ifIframe = ifIframe; + } + + public Boolean getExternal() { + return isExternal; + } + + public void setExternal(Boolean external) { + isExternal = external; + } + + @Override + public String getReference() { + return reference; + } + + public void setReference(String reference) { + this.reference = reference; + } + + public String getResource() { + return resource; + } + + public void setResource(String resource) { + this.resource = resource; + } + + public List getAppInstanceBeans() { + return appInstanceBeans; + } + + public void setAppInstanceBeans(List appInstanceBeans) { + this.appInstanceBeans = appInstanceBeans; + } + + @Override + public String getClassName() { + return className; + } + + @Override + public Resource getAppConnResource() { + return resourceObj; + } + + public void setAppConnResource(Resource resource) { + resourceObj = resource; + } + + public void setClassName(String className) { + this.className = className; + } + + @Override + public String toString() { + return "AppConnBean{" + + "id=" + id + + ", appConnName='" + appConnName + '\'' + + ", className='" + className + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppConnResource.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppConnResource.java new file mode 100644 index 000000000..a698e6bc8 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppConnResource.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.entity; + +import com.webank.wedatasphere.dss.common.entity.Resource; + + +public class AppConnResource { + + private Resource resource; + private long lastModifiedTime; + private long size; + + public Resource getResource() { + return resource; + } + + public void setResource(Resource resource) { + this.resource = resource; + } + + public long getLastModifiedTime() { + return lastModifiedTime; + } + + public void setLastModifiedTime(long lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppInstanceBean.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppInstanceBean.java new file mode 100644 index 000000000..28974baa4 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/entity/AppInstanceBean.java @@ -0,0 +1,103 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.entity; + +import com.webank.wedatasphere.dss.appconn.manager.entity.AppInstanceInfo; + +import java.io.Serializable; + + +public class AppInstanceBean implements AppInstanceInfo, Serializable { + + private static final long serialVersionUID=1L; + + private Long id; + private Long appConnId; + private String label; + private String url; + private String enhanceJson; + private String homepageUri; + + @Override + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getAppConnId() { + return appConnId; + } + + public void setAppConnId(Long appConnId) { + this.appConnId = appConnId; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + @Override + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + @Override + public String getEnhanceJson() { + return enhanceJson; + } + + public void setEnhanceJson(String enhanceJson) { + this.enhanceJson = enhanceJson; + } + + @Override + public String getHomepageUri() { + return homepageUri; + } + + public void setHomepageUri(String homepageUri) { + this.homepageUri = homepageUri; + } + + @Override + public String getLabels() { + return label; + } + + @Override + public String toString() { + return "AppInstanceBean{" + + "id=" + id + + ", appConnId=" + appConnId + + ", label='" + label + '\'' + + ", url='" + url + '\'' + + ", enhanceJson='" + enhanceJson + '\'' + + ", homepageUri='" + homepageUri + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/exception/AppConnNotExistsErrorException.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/exception/AppConnNotExistsErrorException.java new file mode 100644 index 000000000..d73dfcb3c --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/exception/AppConnNotExistsErrorException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + + +public class AppConnNotExistsErrorException extends DSSErrorException { + + public AppConnNotExistsErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public AppConnNotExistsErrorException(int errCode, String desc, Throwable cause) { + super(errCode, desc); + initCause(cause); + } +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/exception/AppConnQualityErrorException.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/exception/AppConnQualityErrorException.java new file mode 100644 index 000000000..7dc1532c2 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/exception/AppConnQualityErrorException.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.framework.appconn.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + +/** + * @author enjoyyin + * @date 2022-04-14 + * @since 0.5.0 + */ +public class AppConnQualityErrorException extends DSSErrorException { + + public AppConnQualityErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public AppConnQualityErrorException(int errCode, String desc, Throwable cause) { + super(errCode, desc); + initCause(cause); + } + +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/restful/AppConnManagerRestfulApi.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/restful/AppConnManagerRestfulApi.java new file mode 100644 index 000000000..0247ced9b --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/restful/AppConnManagerRestfulApi.java @@ -0,0 +1,118 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.restful; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppInstanceInfo; +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnInfoService; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.framework.appconn.conf.AppConnConf; +import com.webank.wedatasphere.dss.framework.appconn.service.AppConnQualityChecker; +import com.webank.wedatasphere.dss.framework.appconn.service.AppConnResourceUploadService; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.PostConstruct; +import java.util.List; + +@RequestMapping(path = "/dss/framework/project/appconn", produces = {"application/json"}) +@RestController +public class AppConnManagerRestfulApi { + private static final Logger LOGGER = LoggerFactory.getLogger(AppConnManagerRestfulApi.class); + + @Autowired + private AppConnInfoService appConnInfoService; + @Autowired + private AppConnResourceUploadService appConnResourceUploadService; + @Autowired + private List appConnQualityCheckers; + + @PostConstruct + public void init() { + if (AppConnConf.IS_APPCONN_MANAGER.getValue()) { + LOGGER.info("First, try to load all AppConn..."); + AppConnManager.getAppConnManager().listAppConns().forEach(appConn -> { + LOGGER.info("Try to check the quality of AppConn {}.", appConn.getAppDesc().getAppName()); + appConnQualityCheckers.forEach(DSSExceptionUtils.handling(checker -> checker.checkQuality(appConn))); + }); + LOGGER.info("All AppConn have loaded successfully."); + LOGGER.info("Last, try to scan AppConn plugins and upload AppConn resources..."); + // reference不为空,说明是引用的其他appconn,不用上传appconn目录 + appConnInfoService.getAppConnInfos().stream().filter(l -> StringUtils.isBlank(l.getReference())) + .forEach(DSSExceptionUtils.handling(appConnInfo -> { + LOGGER.info("Try to scan AppConn {}.", appConnInfo.getAppConnName()); + appConnResourceUploadService.upload(appConnInfo.getAppConnName()); + })); + LOGGER.info("All AppConn plugins has scanned."); + } else { + LOGGER.info("Not appConn manager, will not scan plugins."); + } + } + + @RequestMapping(path ="listAppConnInfos", method = RequestMethod.GET) + public Message listAppConnInfos() { + List appConnInfos = appConnInfoService.getAppConnInfos(); + Message message = Message.ok("Get AppConnInfo list succeed."); + message.data("appConnInfos", appConnInfos); + return message; + } + + @RequestMapping(path ="{appConnName}/get", method = RequestMethod.GET) + public Message get(@PathVariable("appConnName") String appConnName) { + AppConnInfo appConnInfo = appConnInfoService.getAppConnInfo(appConnName); + Message message = Message.ok("Get AppConnInfo succeed."); + message.data("appConnInfo", appConnInfo); + return message; + } + + @RequestMapping(path ="{appConnName}/getAppInstances", method = RequestMethod.GET) + public Message getAppInstancesByAppConnInfo(@PathVariable("appConnName") String appConnName) { + List appInstanceInfos = appConnInfoService.getAppInstancesByAppConnName(appConnName); + Message message = Message.ok("Get AppInstance list succeed."); + message.data("appInstanceInfos", appInstanceInfos); + return message; + } + + @RequestMapping(path ="{appConnName}/load", method = RequestMethod.GET) + public Message load(@PathVariable("appConnName") String appConnName) { + LOGGER.info("Try to reload AppConn {}.", appConnName); + try { + LOGGER.info("First, reload AppConn {}.", appConnName); + AppConnManager.getAppConnManager().reloadAppConn(appConnName); + AppConn appConn = AppConnManager.getAppConnManager().getAppConn(appConnName); + LOGGER.info("Second, check the quality of AppConn {}.", appConnName); + appConnQualityCheckers.forEach(DSSExceptionUtils.handling(checker -> checker.checkQuality(appConn))); + LOGGER.info("Last, upload AppConn {} resources.", appConnName); + appConnResourceUploadService.upload(appConnName); + } catch (Exception e) { + LOGGER.error("Load AppConn " + appConnName + " failed.", e); + return Message.error("Load AppConn " + appConnName + " failed. Reason: " + ExceptionUtils.getRootCauseMessage(e)); + } + return Message.ok("Load AppConn " + appConnName + " succeed."); + } + +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/AppConnQualityChecker.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/AppConnQualityChecker.java new file mode 100644 index 000000000..5bcd41747 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/AppConnQualityChecker.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.framework.appconn.service; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.framework.appconn.exception.AppConnQualityErrorException; + +/** + * @author enjoyyin + * @date 2022-04-14 + * @since 0.5.0 + */ +public interface AppConnQualityChecker { + + /** + * 检查用户实现的 AppConn 是否存在质量问题 + * @throws AppConnQualityErrorException 如果存在质量问题,请抛出该异常 + */ + void checkQuality(AppConn appConn) throws AppConnQualityErrorException; + +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/AppConnResourceUploadService.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/AppConnResourceUploadService.java new file mode 100644 index 000000000..f97951b77 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/AppConnResourceUploadService.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + + +public interface AppConnResourceUploadService { + + void upload(String appConnName) throws DSSErrorException; + +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AbstractAppConnQualityChecker.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AbstractAppConnQualityChecker.java new file mode 100644 index 000000000..787f9a289 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AbstractAppConnQualityChecker.java @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.dss.framework.appconn.service.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.framework.appconn.exception.AppConnQualityErrorException; +import com.webank.wedatasphere.dss.framework.appconn.service.AppConnQualityChecker; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-04-14 + * @since 1.1.0 + */ +public abstract class AbstractAppConnQualityChecker implements AppConnQualityChecker { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + private List ignoreAppConnNameList; + + public AbstractAppConnQualityChecker(List ignoreAppConnNameList) { + this.ignoreAppConnNameList = ignoreAppConnNameList; + } + + public AbstractAppConnQualityChecker(String ignoreAppConnNameStr) { + if(StringUtils.isNotBlank(ignoreAppConnNameStr)) { + this.ignoreAppConnNameList = Arrays.asList(ignoreAppConnNameStr.split(",")); + } + } + + @Override + public void checkQuality(AppConn appConn) throws AppConnQualityErrorException { + if(ignoreAppConnNameList != null && ignoreAppConnNameList.contains(appConn.getAppDesc().getAppName())) { + logger.info("ignore the quality checker of AppConn {}.", appConn.getAppDesc().getAppName()); + return; + } + checkAppConnQuality(appConn); + } + + protected abstract void checkAppConnQuality(AppConn appConn) throws AppConnQualityErrorException; + + protected void checkAppInstance(AppConn appConn) throws AppConnQualityErrorException { + if(appConn.getAppDesc().getAppInstances().isEmpty()) { + throw new AppConnQualityErrorException(10005, getErrorMsg(appConn.getAppDesc().getAppName(), + "no appInstance is found in DSS database.")); + } + } + + protected void checkNull(Object object, String appConnName, String objectStr) throws AppConnQualityErrorException { + if(object == null) { + throw new AppConnQualityErrorException(10005, errorMsg(appConnName, objectStr)); + } + } + + protected void checkBoolean(Boolean condition, String appConnName, String reason) throws AppConnQualityErrorException { + if(condition) { + throw new AppConnQualityErrorException(10005, getErrorMsg(appConnName, reason)); + } + } + + protected String errorMsg(String appConnName, String object) { + return getErrorMsg(appConnName, object + " is not exists."); + } + + protected String getErrorMsg(String appConnName, String reason) { + return String.format("check the quality of AppConn %s failed, reason: %s.", appConnName, reason); + } + +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AppConnInfoServiceImpl.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AppConnInfoServiceImpl.java new file mode 100644 index 000000000..4a40e0b3c --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AppConnInfoServiceImpl.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.service.impl; + +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppInstanceInfo; +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnInfoService; +import com.webank.wedatasphere.dss.framework.appconn.conf.AppConnConf; +import com.webank.wedatasphere.dss.framework.appconn.dao.AppConnMapper; +import com.webank.wedatasphere.dss.framework.appconn.dao.AppInstanceMapper; +import com.webank.wedatasphere.dss.framework.appconn.entity.AppConnBean; +import com.webank.wedatasphere.dss.framework.appconn.utils.AppConnServiceUtils; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class AppConnInfoServiceImpl implements AppConnInfoService { + @Autowired + private AppConnMapper appConnMapper; + @Autowired + private AppInstanceMapper appInstanceMapper; + + @Override + public List getAppConnInfos() { + List appConnBeans = appConnMapper.getAllAppConnBeans(); + appConnBeans.stream().filter(appConnBean -> !AppConnConf.DISABLED_APP_CONNS.contains(appConnBean.getAppConnName())) + .forEach(appConnBean -> { + String resource = appConnBean.getResource(); + if(StringUtils.isNotBlank(resource)) { + appConnBean.setAppConnResource(AppConnServiceUtils.stringToResource(resource).getResource()); + } + }); + return appConnBeans; + } + + @Override + public AppConnInfo getAppConnInfo(String appConnName) { + return appConnMapper.getAppConnBeanByName(appConnName); + } + + @Override + public List getAppInstancesByAppConnInfo(AppConnInfo appConnInfo) { + Long id = ((AppConnBean) appConnInfo).getId(); + return appInstanceMapper.getAppInstancesByAppConnId(id); + } + + @Override + public List getAppInstancesByAppConnName(String appConnName) { + AppConnBean appConnBean = appConnMapper.getAppConnBeanByName(appConnName); + return getAppInstancesByAppConnInfo(appConnBean); + } +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AppConnResourceServiceImpl.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AppConnResourceServiceImpl.java new file mode 100644 index 000000000..b422a79de --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/AppConnResourceServiceImpl.java @@ -0,0 +1,162 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.service.impl; + +import com.webank.wedatasphere.dss.appconn.loader.utils.AppConnUtils; +import com.webank.wedatasphere.dss.appconn.manager.entity.AppConnInfo; +import com.webank.wedatasphere.dss.appconn.manager.service.AppConnResourceService; +import com.webank.wedatasphere.dss.appconn.manager.utils.AppConnIndexFileUtils; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.framework.appconn.dao.AppConnMapper; +import com.webank.wedatasphere.dss.framework.appconn.entity.AppConnBean; +import com.webank.wedatasphere.dss.framework.appconn.entity.AppConnResource; +import com.webank.wedatasphere.dss.framework.appconn.exception.AppConnNotExistsErrorException; +import com.webank.wedatasphere.dss.framework.appconn.service.AppConnResourceUploadService; +import com.webank.wedatasphere.dss.framework.appconn.utils.AppConnServiceUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.bml.protocol.BmlUpdateResponse; +import org.apache.linkis.bml.protocol.BmlUploadResponse; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.*; +import java.nio.file.Paths; + + +@Component +public class AppConnResourceServiceImpl implements AppConnResourceService, AppConnResourceUploadService { + + private Logger LOGGER = LoggerFactory.getLogger(AppConnResourceServiceImpl.class); + private static final String LIB_NAME = "/lib"; + + @Autowired + private AppConnMapper appConnMapper; + private BmlClient bmlClient; + + @PostConstruct + public void init() { + bmlClient = BmlClientFactory.createBmlClient(); + } + + @Override + public String getAppConnHome(AppConnInfo appConnInfo) { + return Paths.get(AppConnUtils.getAppConnHomePath(), appConnInfo.getAppConnName()).toFile().getPath(); + } + + @Override + public void upload(String appConnName) throws DSSErrorException { + File appConnPath = new File(AppConnUtils.getAppConnHomePath(), appConnName); + if (!appConnPath.exists()) { + throw new AppConnNotExistsErrorException(20350, "AppConn home path " + appConnPath.getPath() + " not exists."); + } else if (!appConnPath.isDirectory()) { + throw new AppConnNotExistsErrorException(20350, "AppConn home path " + appConnPath.getPath() + " is not a directory."); + } + File zipFile = new File(appConnPath.getPath() + ".zip"); + if (zipFile.exists() && !zipFile.delete()) { + throw new AppConnNotExistsErrorException(20001, "No permission to delete old zip file " + zipFile); + } + ZipHelper.zip(appConnPath.getPath(), false); + AppConnBean appConnBean = appConnMapper.getAppConnBeanByName(appConnName); + AppConnResource appConnResource; + File indexFile = null; + if (appConnBean == null) { + throw new AppConnNotExistsErrorException(20350, "AppConn not exists in DB, please update db at first."); + } else if (StringUtils.isNotBlank(appConnBean.getResource())) { + // If resource is exists, then indexFile is also exists. + appConnResource = AppConnServiceUtils.stringToResource(appConnBean.getResource()); + indexFile = AppConnIndexFileUtils.getIndexFile(appConnPath); + if (appConnPath.lastModified() == appConnResource.getLastModifiedTime() + && zipFile.length() == appConnResource.getSize() && AppConnIndexFileUtils.isLatestIndex(appConnPath, appConnResource.getResource())) { + LOGGER.info("No necessary to update the AppConn {}, since it's packages has no changes in path {}.", appConnName, appConnPath.getPath()); + return; + } + } else { + // If resource is not exists, this is the first time to upload this AppConn. + appConnResource = new AppConnResource(); + } + // At first, upload appConn file to BML + Resource resource = new Resource(); + InputStream inputStream = null; + if (appConnResource.getResource() != null) { + try { + inputStream = new FileInputStream(zipFile.getPath()); + BmlUpdateResponse response = bmlClient.updateResource(Utils.getJvmUser(), appConnResource.getResource().getResourceId(), zipFile.getPath(),inputStream); + resource.setResourceId(appConnResource.getResource().getResourceId()); + resource.setVersion(response.version()); + } catch (FileNotFoundException e) { + throw new AppConnNotExistsErrorException(20351, "AppConn update to bml failed"+e.getMessage()); + } finally { + IOUtils.closeQuietly(inputStream); + } + LOGGER.info("AppConn {} updated Resource, from {} to {}.", appConnName, + DSSCommonUtils.COMMON_GSON.toJson(appConnResource.getResource()), DSSCommonUtils.COMMON_GSON.toJson(resource)); + } else { + try { + inputStream = new FileInputStream(zipFile.getPath()); + BmlUploadResponse response = bmlClient.uploadResource(Utils.getJvmUser(), zipFile.getPath(), inputStream); + resource.setResourceId(response.resourceId()); + resource.setVersion(response.version()); + } catch (FileNotFoundException e) { + throw new AppConnNotExistsErrorException(20352, "AppConn update to bml failed"+e.getMessage()); + } finally { + IOUtils.closeQuietly(inputStream); + } + LOGGER.info("AppConn {} completed the first upload of Resource with {}.", appConnName, + DSSCommonUtils.COMMON_GSON.toJson(resource)); + } + resource.setFileName(zipFile.getName()); + // Then, insert into db. + appConnResource.setLastModifiedTime(appConnPath.lastModified()); + appConnResource.setSize(zipFile.length()); + appConnResource.setResource(resource); + String resourceStr = AppConnServiceUtils.resourceToString(appConnResource); + + AppConnBean appConnBeanReLoad = new AppConnBean(); + appConnBeanReLoad.setId(appConnBean.getId()); + appConnBeanReLoad.setResource(resourceStr); + appConnBeanReLoad.setAppConnName(appConnName); + appConnBeanReLoad.setClassName(appConnBean.getClassName()); + appConnMapper.updateResourceByName(appConnBeanReLoad); + // update index file. + if (indexFile != null && !indexFile.delete()) { + throw new AppConnNotExistsErrorException(20350, "Delete index file " + indexFile.getName() + " failed, please ensure the permission is all right."); + } + indexFile = new File(appConnPath, AppConnIndexFileUtils.getIndexFileName(resource)); + try { + indexFile.createNewFile(); + } catch (IOException e) { + throw new AppConnNotExistsErrorException(20350, "create index file " + indexFile.getName() + " failed, please ensure the permission is all right.", e); + } + LOGGER.info("AppConn {} has updated resource to {}.", appConnName, resourceStr); + } + + @PreDestroy + public void destroy() { + IOUtils.closeQuietly(bmlClient); + } +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/DevelopmentAppConnQualityChecker.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/DevelopmentAppConnQualityChecker.java new file mode 100644 index 000000000..d9f1d92ca --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/DevelopmentAppConnQualityChecker.java @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.dss.framework.appconn.service.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.framework.appconn.conf.AppConnConf; +import com.webank.wedatasphere.dss.framework.appconn.exception.AppConnQualityErrorException; +import com.webank.wedatasphere.dss.standard.app.development.service.RefCRUDService; +import com.webank.wedatasphere.dss.standard.app.development.service.RefExecutionService; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import org.springframework.stereotype.Component; + +/** + * @author enjoyyin + * @date 2022-04-14 + * @since 1.1.0 + */ +@Component +public class DevelopmentAppConnQualityChecker extends AbstractAppConnQualityChecker { + + public DevelopmentAppConnQualityChecker() { + super(AppConnConf.DEVELOPMENT_QUALITY_CHECKER_IGNORE_LIST.getValue()); + } + + @Override + protected void checkAppConnQuality(AppConn appConn) throws AppConnQualityErrorException { + if(!(appConn instanceof OnlyDevelopmentAppConn)) { + return; + } + String appConnName = appConn.getAppDesc().getAppName(); + checkAppInstance(appConn); + AppInstance appInstance = appConn.getAppDesc().getAppInstances().get(0); + DevelopmentIntegrationStandard developmentIntegrationStandard = ((OnlyDevelopmentAppConn) appConn).getOrCreateDevelopmentStandard(); + checkNull(developmentIntegrationStandard, appConnName, "developmentStandard"); + RefExecutionService refExecutionService = developmentIntegrationStandard.getRefExecutionService(appInstance); + checkNull(refExecutionService, appConnName, "refExecutionService"); + checkNull(refExecutionService.getRefExecutionOperation(), appConnName, "refExecutionOperation"); + checkBoolean(developmentIntegrationStandard.getRefCRUDService(appInstance) != null && + developmentIntegrationStandard.getRefImportService(appInstance) == null, appConnName, + "RefImportService is needed since refCRUDService is exists."); + checkBoolean(developmentIntegrationStandard.getRefCRUDService(appInstance) != null && + developmentIntegrationStandard.getRefExportService(appInstance) == null, appConnName, + "RefExportService is needed since refCRUDService is exists."); + RefCRUDService refCRUDService = developmentIntegrationStandard.getRefCRUDService(appInstance); + if(refCRUDService == null) { + return; + } + checkNull(refCRUDService.getRefUpdateOperation(), appConnName, "refUpdateOperation"); + checkNull(refCRUDService.getRefCopyOperation(), appConnName, "refCopyOperation"); + checkNull(refCRUDService.getRefDeletionOperation(), appConnName, "refDeletionOperation"); + checkNull(developmentIntegrationStandard.getRefImportService(appInstance).getRefImportOperation(), appConnName, "refImportOperation"); + checkNull(developmentIntegrationStandard.getRefExportService(appInstance).getRefExportOperation(), appConnName, "refExportOperation"); + } + +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/ProjectAppConnQualityChecker.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/ProjectAppConnQualityChecker.java new file mode 100644 index 000000000..1b79eb95c --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/service/impl/ProjectAppConnQualityChecker.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.dss.framework.appconn.service.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyStructureAppConn; +import com.webank.wedatasphere.dss.framework.appconn.conf.AppConnConf; +import com.webank.wedatasphere.dss.framework.appconn.exception.AppConnQualityErrorException; +import com.webank.wedatasphere.dss.standard.app.structure.StructureIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import org.springframework.stereotype.Component; + +/** + * @author enjoyyin + * @date 2022-04-14 + * @since 1.1.0 + */ +@Component +public class ProjectAppConnQualityChecker extends AbstractAppConnQualityChecker { + + public ProjectAppConnQualityChecker() { + super(AppConnConf.PROJECT_QUALITY_CHECKER_IGNORE_LIST.getValue()); + } + + @Override + protected void checkAppConnQuality(AppConn appConn) throws AppConnQualityErrorException { + if(!(appConn instanceof OnlyStructureAppConn)) { + return; + } + String appConnName = appConn.getAppDesc().getAppName(); + checkAppInstance(appConn); + AppInstance appInstance = appConn.getAppDesc().getAppInstances().get(0); + OnlyStructureAppConn onlyStructureAppConn = (OnlyStructureAppConn) appConn; + StructureIntegrationStandard structureIntegrationStandard = onlyStructureAppConn.getOrCreateStructureStandard(); + checkNull(structureIntegrationStandard, appConnName, "structureStandard"); + ProjectService projectService = structureIntegrationStandard.getProjectService(appInstance); + checkNull(projectService, appConnName, "projectService"); + checkBoolean(projectService.isProjectNameUnique() && projectService.getProjectSearchOperation() == null, + appConnName, "isProjectNameUnique is true but projectSearchOperation is not exists."); + checkNull(projectService.getProjectCreationOperation(), appConnName, "projectCreationOperation"); + checkNull(projectService.getProjectUpdateOperation(), appConnName, "projectUpdateOperation"); + checkNull(projectService.getProjectDeletionOperation(), appConnName, "projectDeletionOperation"); + } + + +} diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/utils/AppConnServiceUtils.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/utils/AppConnServiceUtils.java new file mode 100644 index 000000000..11d851cd8 --- /dev/null +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/utils/AppConnServiceUtils.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.appconn.utils; + +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.framework.appconn.entity.AppConnResource; + + +public class AppConnServiceUtils { + + public static String resourceToString(AppConnResource resource) { + return DSSCommonUtils.COMMON_GSON.toJson(resource); + } + + public static AppConnResource stringToResource(String resource) { + return DSSCommonUtils.COMMON_GSON.fromJson(resource, AppConnResource.class); + } + +} diff --git a/dss-framework/dss-framework-admin-service/pom.xml b/dss-framework/dss-framework-admin-service/pom.xml new file mode 100644 index 000000000..7bf00155c --- /dev/null +++ b/dss-framework/dss-framework-admin-service/pom.xml @@ -0,0 +1,145 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + + + dss-framework-admin-service + 4.0.0 + + + 2.9.2 + 1.9.2 + 3.9 + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + provided + + + org.apache.linkis + linkis-httpclient + ${linkis.version} + + + linkis-common + org.apache.linkis + + + + + + com.webank.wedatasphere.dss + dss-framework-common + ${dss.version} + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-appconn-manager-client + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + provided + + + + org.apache.commons + commons-math3 + provided + + + org.projectlombok + lombok + 1.18.16 + compile + + + + com.squareup.okhttp + okhttp + 2.7.5 + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + + + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/constant/UserConstants.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/constant/UserConstants.java new file mode 100644 index 000000000..6ad62cb9e --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/constant/UserConstants.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.framework.admin.common.constant; + +public class UserConstants { + /** 正常状态 */ + public static final String NORMAL = "0"; + + /** 异常状态 */ + public static final String EXCEPTION = "1"; + + /** 用户封禁状态 */ + public static final String USER_DISABLE = "1"; + + /** 角色封禁状态 */ + public static final String ROLE_DISABLE = "1"; + + /** 部门正常状态 */ + public static final String DEPT_NORMAL = "0"; + + /** 部门停用状态 */ + public static final String DEPT_DISABLE = "1"; + + /** 空格 */ + public static final String SINGLE_SPACE = " "; + + + /** 校验返回结果码 */ + public final static String UNIQUE = "0"; + + public final static String NOT_UNIQUE = "1"; + + /** + * 用户名长度限制 + */ + public static final int USERNAME_MIN_LENGTH = 2; + + public static final int USERNAME_MAX_LENGTH = 20; + + /** + * 密码长度限制 + */ + public static final int PASSWORD_MIN_LENGTH = 5; + + public static final int PASSWORD_MAX_LENGTH = 20; +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/BaseEntity.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/BaseEntity.java new file mode 100644 index 000000000..7cccd6667 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/BaseEntity.java @@ -0,0 +1,109 @@ +package com.webank.wedatasphere.dss.framework.admin.common.domain; + + +import java.io.Serializable; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +public class BaseEntity implements Serializable +{ + private static final long serialVersionUID = 1L; + + /** 搜索值 */ + private String searchValue; + + /** 创建者 */ + private String createBy; + + /** 创建时间 */ +// @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + + /** 更新者 */ + private String updateBy; + + /** 更新时间 */ +// @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date updateTime; + + /** 备注 */ + private String remark; + + /** 请求参数 */ + private Map params; + + public String getSearchValue() + { + return searchValue; + } + + public void setSearchValue(String searchValue) + { + this.searchValue = searchValue; + } + + public String getCreateBy() + { + return createBy; + } + + public void setCreateBy(String createBy) + { + this.createBy = createBy; + } + + public Date getCreateTime() + { + return createTime; + } + + public void setCreateTime(Date createTime) + { + this.createTime = createTime; + } + + public String getUpdateBy() + { + return updateBy; + } + + public void setUpdateBy(String updateBy) + { + this.updateBy = updateBy; + } + + public Date getUpdateTime() + { + return updateTime; + } + + public void setUpdateTime(Date updateTime) + { + this.updateTime = updateTime; + } + + public String getRemark() + { + return remark; + } + + public void setRemark(String remark) + { + this.remark = remark; + } + + public Map getParams() + { + if (params == null) + { + params = new HashMap<>(); + } + return params; + } + + public void setParams(Map params) + { + this.params = params; + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/Message.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/Message.java new file mode 100644 index 000000000..074b85b03 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/Message.java @@ -0,0 +1,133 @@ +package com.webank.wedatasphere.dss.framework.admin.common.domain; + +//import lombok.Data; + +import java.util.HashMap; +import java.util.Map; +//@Data +public class Message { + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Map getData() { + return data; + } + + public void setData(Map data) { + this.data = data; + } + + private String method; + private Integer status; + private String message; + private Map data = new HashMap<>(); + + public Message() { + } + + + /** + * 返回成功结果 + * + * @return + */ + public static Message ok() { + Message message = new Message(); + message.setStatus(ResponseEnum.SUCCESS.getStatus()); + message.setMessage(ResponseEnum.SUCCESS.getMessage()); + return message; + } + + + /** + * 返回失败结果 + * + * @return + */ + public static Message error() { + Message message = new Message(); + message.setStatus(ResponseEnum.ERROR.getStatus()); + message.setMessage(ResponseEnum.ERROR.getMessage()); + return message; + } + + /** + * 返回特定结果 + * + * @param responseEnum + * @return + */ + public static Message setResult(ResponseEnum responseEnum) { + Message message = new Message(); + message.setStatus(responseEnum.getStatus()); + message.setMessage(responseEnum.getMessage()); + return message; + } + + /** + * 封装返回数据 + * + * @param key + * @param value + * @return + */ + public Message data(String key, Object value) { + this.data.put(key, value); + return this; + } + + /** + * 封装返回数据,若为Map集合,则直接赋值即可 + * + * @param map + * @return + */ + public Message data(Map map) { + this.setData(map); + return this; + } + + /** + * 设置特定的消息 + * + * @param message + * @return + */ + public Message message(String message) { + this.setMessage(message); + return this; + } + + /** + * 设置特定的响应码 + * + * @param code + * @return + */ + public Message status(Integer code) { + this.setStatus(code); + return this; + } + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PageDomain.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PageDomain.java new file mode 100644 index 000000000..7d0eb2bfa --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PageDomain.java @@ -0,0 +1,65 @@ +package com.webank.wedatasphere.dss.framework.admin.common.domain; + +import com.webank.wedatasphere.dss.framework.admin.common.utils.StringUtils; + +public class PageDomain { + + /** + * 当前记录起始索引 + */ + private Integer pageNum; + + /** + * 每页显示记录数 + */ + private Integer pageSize; + + /** + * 排序列 + */ + private String orderByColumn; + + /** + * 排序的方向desc或者asc + */ + private String isAsc = "asc"; + + public String getOrderBy() { + if (StringUtils.isEmpty(orderByColumn)) { + return ""; + } + return StringUtils.toUnderScoreCase(orderByColumn) + " " + isAsc; + } + + public Integer getPageNum() { + return pageNum; + } + + public void setPageNum(Integer pageNum) { + this.pageNum = pageNum; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + public String getOrderByColumn() { + return orderByColumn; + } + + public void setOrderByColumn(String orderByColumn) { + this.orderByColumn = orderByColumn; + } + + public String getIsAsc() { + return isAsc; + } + + public void setIsAsc(String isAsc) { + this.isAsc = isAsc; + } + } diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PasswordResult.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PasswordResult.java new file mode 100644 index 000000000..726f03cb6 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PasswordResult.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.framework.admin.common.domain; + + +public enum PasswordResult { + + PASSWORD_RULE_PASS("11000", "密码校验通过"), + + PASSWORD_STRENGTH_ERROR("11002", "密码应包括数字、小写字母、大写字母和特殊符号四种类型字符(长度为8-26位)"), + + PASSWORD_RELATED_USERNAME_ERROR("11003", "密码应与用户名无关"), + + PASSWORD_STARTER_ERROR("11004", "密码须以字母开头"), + + PASSWORD_WITH_DICT_WORD_ERROR("11005", "密码中不应出现弱密码字典中的禁用字段"), + + PASSWORD_KEYBOARD_CONTINUOUS_ERROR("11006", "密码中不应出现键盘序"); + + private final String code; + + private final String message; + + public String getCode() { + return code; + } + + public String getMessage() { + return message; + } + + + + PasswordResult(String code, String message) { + this.code = code; + this.message = message; + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PositionBo.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PositionBo.java new file mode 100644 index 000000000..fa3e97c1a --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/PositionBo.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.dss.framework.admin.common.domain; + + +public class PositionBo { + /** + * 行号 + */ + private int row; + + /** + * 列号 + */ + private int column; + + public PositionBo(Integer row, Integer column) { + this.row = row; + this.column = column; + } + + public PositionBo() { + } + + @Override + public String toString() { + return "PositionBo{" + + "row=" + row + + ", column=" + column + + '}'; + } + + public int getRow() { + return row; + } + + public void setRow(int row) { + this.row = row; + } + + public int getColumn() { + return column; + } + + public void setColumn(int column) { + this.column = column; + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/ResponseEnum.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/ResponseEnum.java new file mode 100644 index 000000000..510c77065 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/ResponseEnum.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.framework.admin.common.domain; + +//import lombok.AllArgsConstructor; +//import lombok.Getter; +//import lombok.ToString; + +//@Getter +//@ToString +//@AllArgsConstructor +public enum ResponseEnum { + /* + case -1 => 401 + case 0 => 200 + case 1 => 400 + case 2 => 412 + case 3 => 403 + case 4 => 206*/ + //-1 no login, 0 success, 1 error, 2 validate failed, 3 auth failed, 4 warning + NO_LOGIN(-1, "no login"), + SUCCESS(0, "success"), + ERROR(1, "error"), + VALIDATE_FAILED(2, "validate failed"), + AUTH_FAILED(3, "auth failed"), + WARNING(4,"warning"); + + ResponseEnum(Integer status, String message) { + this.status = status; + this.message = message; + } + + @Override + public String toString() { + return "ResponseEnum{" + + "status=" + status + + ", message='" + message + '\'' + + '}'; + } + + private Integer status; + private String message; + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/TableDataInfo.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/TableDataInfo.java new file mode 100644 index 000000000..ad80f4e97 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/TableDataInfo.java @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.dss.framework.admin.common.domain; + +import java.io.Serializable; +import java.util.HashMap; + +public class TableDataInfo implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * 总记录数 + */ + private long total; + + /** + * 列表数据 + */ + private HashMap data; + + /** + * 消息状态码 + */ + private int status; + + /** + * 消息内容 + */ + private String message; + + /** + * 表格数据对象 + */ + public TableDataInfo() { + } + + /** + * 分页 + * + * @param data 列表数据 + * @param total 总记录数 + */ + public TableDataInfo(HashMap data, int total) { + this.data = data; + this.total = total; + } + + + + public HashMap getData() { + return data; + } + + public void setData(HashMap data) { + this.data = data; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/TreeEntity.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/TreeEntity.java new file mode 100644 index 000000000..139b1a9fb --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/domain/TreeEntity.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.dss.framework.admin.common.domain; + + +import java.util.ArrayList; +import java.util.List; + + +public class TreeEntity extends BaseEntity +{ + private static final long serialVersionUID = 1L; + + /** 父菜单名称 */ + private String parentName; + + /** 父菜单ID */ + private Long parentId; + + /** 显示顺序 */ + private Integer orderNum; + + /** 祖级列表 */ + private String ancestors; + + /** 子部门 */ + private List children = new ArrayList<>(); + + public String getParentName() + { + return parentName; + } + + public void setParentName(String parentName) + { + this.parentName = parentName; + } + + public Long getParentId() + { + return parentId; + } + + public void setParentId(Long parentId) + { + this.parentId = parentId; + } + + public Integer getOrderNum() + { + return orderNum; + } + + public void setOrderNum(Integer orderNum) + { + this.orderNum = orderNum; + } + + public String getAncestors() + { + return ancestors; + } + + public void setAncestors(String ancestors) + { + this.ancestors = ancestors; + } + + public List getChildren() + { + return children; + } + + public void setChildren(List children) + { + this.children = children; + } +} + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/exception/AdminException.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/exception/AdminException.java new file mode 100644 index 000000000..6eb4da3db --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/exception/AdminException.java @@ -0,0 +1,82 @@ +package com.webank.wedatasphere.dss.framework.admin.common.exception; + + +import com.webank.wedatasphere.dss.framework.admin.common.domain.ResponseEnum; +//import lombok.Data; +//import lombok.NoArgsConstructor; + +//@Data +//@NoArgsConstructor +public class AdminException extends RuntimeException { + + public Integer getCode() { + return code; + } + + public void setCode(Integer code) { + this.code = code; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + /** + * 错误码 + */ + private Integer code; + /** + * 错误信息 + */ + private String message; + + /** + * @param message 错误消息 + */ + public AdminException(String message) { + this.message = message; + } + + /** + * @param message 错误消息 + * @param code 错误码 + */ + public AdminException(String message, Integer code) { + this.message = message; + this.code = code; + } + + /** + * @param message 错误消息 + * @param code 错误码 + * @param cause 原始异常对象 + */ + public AdminException(String message, Integer code, Throwable cause) { + super(cause); + this.message = message; + this.code = code; + } + + /** + * @param resultCodeEnum 接收枚举类型 + */ + public AdminException(ResponseEnum resultCodeEnum) { + this.message = resultCodeEnum.getMessage(); + this.code = resultCodeEnum.getStatus(); + } + + /** + * @param resultCodeEnum 接收枚举类型 + * @param cause 原始异常对象 + */ + public AdminException(ResponseEnum resultCodeEnum, Throwable cause) { + super(cause); + this.message = resultCodeEnum.getMessage(); + this.code = resultCodeEnum.getStatus(); + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/PasswordUtils.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/PasswordUtils.java new file mode 100644 index 000000000..3e18d2d18 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/PasswordUtils.java @@ -0,0 +1,129 @@ +package com.webank.wedatasphere.dss.framework.admin.common.utils; + +import com.webank.wedatasphere.dss.framework.admin.common.domain.PasswordResult; +import com.webank.wedatasphere.dss.framework.admin.common.domain.PositionBo; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminUser; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; + +import java.util.ArrayList; +import java.util.List; + +public class PasswordUtils { + /** + * 密码强度正则匹配 + */ + private static final String PWD_STRENGTH_REGEX + = "^(?=.*[a-z])(?=.*[A-Z])(?=.*[0-9])(?=.*[@$!%*#_~?&^])[A-Za-z0-9@$!%*#_~?&^]{8,26}$"; + + /** + * 密码以字母开头 + */ + private static final String PWD_STARTER_REGEX = "^[a-zA-z].*"; + /** + * 字符表行数 + */ + private static final int CHAR_TABLE_ROW_NUM = 4; + /** + * 字符表列数 + */ + private static final int CHAR_TABLE_COLUMN_NUM = 13; + /** + * 常规字符表 + */ + private static final char[][] charTable = {{'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '-', '=', '\0'}, + {'q', 'w', 'e', 'r', 't', 'y', 'u', 'i', 'o', 'p', '[', ']', '\\'}, + {'a', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l', ';', '\'', '\0', '\0'}, + {'z', 'x', 'c', 'v', 'b', 'n', 'm', ',', '.', '/', '\0', '\0', '\0'}}; + /** + * shift下的字符表 + */ + private static final char[][] charTableWithShift = { + {'!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '_', '+', '\0'}, + {'q', 'w', 'e', 'r', 't', 'y', 'u', 'i', 'o', 'p', '{', '}', '|'}, + {'a', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l', ':', '"', '\0', '\0'}, + {'z', 'x', 'c', 'v', 'b', 'n', 'm', '<', '>', '?', '\0', '\0', '\0'}}; + /** + * 加密编码 + */ + private static final BCryptPasswordEncoder bCryptPasswordEncoder = new BCryptPasswordEncoder(); + + private PasswordUtils() { + } + + /** + * 密码规则 + * + * @return + */ + public static PasswordResult checkPwd(String password, DssAdminUser user) { + + // 1.数字、大写字母、小写字母、特殊符号的组合,长度为8-26位 + if (!StringUtils.hasText(password) || !password.matches(PWD_STRENGTH_REGEX)) { + return PasswordResult.PASSWORD_STRENGTH_ERROR; + } + // 2.排除与用户名相关 + else if (StringUtils.hasText(user.getUserName()) && password.contains(user.getUserName())) { + return PasswordResult.PASSWORD_RELATED_USERNAME_ERROR; + } + // 3.密码须以字母开头 + else if (!StringUtils.hasText(password) || !password.matches(PWD_STARTER_REGEX)) { + return PasswordResult.PASSWORD_STARTER_ERROR; + } + // 4.排除键盘序 + else if (isKeyBoardContinuous(password)) { + return PasswordResult.PASSWORD_KEYBOARD_CONTINUOUS_ERROR; + } + + + return PasswordResult.PASSWORD_RULE_PASS; + } + + /** + * 键盘序检测 + */ + private static boolean isKeyBoardContinuous(String password) { + List positions = new ArrayList<>(); + + for (int i = 0; i < password.length(); ++i) { + PositionBo position = getPositionFromKeyBoard(password.toLowerCase().charAt(i)); + position = (position == null) ? new PositionBo(-1, -1) : position; + positions.add(position); + } + + for (int i = 1; i < password.length() - 1; ++i) { + PositionBo front = positions.get(i - 1); + PositionBo middle = positions.get(i); + PositionBo end = positions.get(i + 1); + // 正向连续(asd)或者反向连续(dsa) + if (front.getRow() == middle.getRow() && middle.getRow() == end.getRow()) { + if (front.getColumn() + 1 == middle.getColumn() && middle.getColumn() + 1 == end.getColumn()) { + return true; + } else if (front.getColumn() - 1 == middle.getColumn() && middle.getColumn() - 1 == end.getColumn()) { + return true; + } + } + // 正向连续(qaz)或者反向连续(zaq) + if (front.getColumn() == middle.getColumn() && middle.getColumn() == end.getColumn()) { + if (front.getRow() + 1 == middle.getRow() && middle.getRow() + 1 == end.getRow()) { + return true; + } else if (front.getRow() - 1 == middle.getRow() && middle.getRow() - 1 == end.getRow()) { + return true; + } + } + } + return false; + } + + private static PositionBo getPositionFromKeyBoard(char ch) { + for (int row = 0; row < CHAR_TABLE_ROW_NUM; ++row) { + for (int column = 0; column < CHAR_TABLE_COLUMN_NUM; ++column) { + if (ch == charTable[row][column] || ch == charTableWithShift[row][column]) { + return new PositionBo(row, column); + } + } + } + return null; + } + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/SecurityUtils.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/SecurityUtils.java new file mode 100644 index 000000000..a59924016 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/SecurityUtils.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.framework.admin.common.utils; + +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; + +public class SecurityUtils { + /** + * 生成BCryptPasswordEncoder密码 + * + * @param password 密码 + * @return 加密字符串 + */ + public static String encryptPassword(String password) + { + BCryptPasswordEncoder passwordEncoder = new BCryptPasswordEncoder(); + return passwordEncoder.encode(password); + } + + /** + * 判断密码是否相同 + * + * @param rawPassword 真实密码 + * @param encodedPassword 加密后字符 + * @return 结果 + */ + public static boolean matchesPassword(String rawPassword, String encodedPassword) + { + BCryptPasswordEncoder passwordEncoder = new BCryptPasswordEncoder(); + return passwordEncoder.matches(rawPassword, encodedPassword); + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/ServletUtils.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/ServletUtils.java new file mode 100644 index 000000000..2c265f004 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/ServletUtils.java @@ -0,0 +1,174 @@ +package com.webank.wedatasphere.dss.framework.admin.common.utils; + +import org.springframework.web.context.request.RequestAttributes; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.Map; + +public class ServletUtils +{ + public static String toStr(Object value, String defaultValue) + { + if (null == value) + { + return defaultValue; + } + if (value instanceof String) + { + return (String) value; + } + return value.toString(); + } + + /** + * 获取String参数 + */ + public static String getParameter(String name) + { + return getRequest().getParameter(name); + } + + /** + * 获取String参数 + */ + public static String getParameter(String name, String defaultValue) + { + return toStr(getRequest().getParameter(name), defaultValue); + } + + /** + * 获取Integer参数 + */ + public static Integer getParameterToInt(String name) + { + return toInt(getRequest().getParameter(name)); + } + + /** + * 获取Integer参数 + */ + public static Integer getParameterToInt(String name, Integer defaultValue) + { + return toInt(getRequest().getParameter(name), defaultValue); + } + + /** + * 获取request + */ + public static HttpServletRequest getRequest() + { + try + { + return getRequestAttributes().getRequest(); + } + catch (Exception e) + { + return null; + } + } + + /** + * 获取response + */ + public static HttpServletResponse getResponse() + { + try + { + return getRequestAttributes().getResponse(); + } + catch (Exception e) + { + return null; + } + } + + /** + * 获取session + */ + public static HttpSession getSession() + { + return getRequest().getSession(); + } + + public static ServletRequestAttributes getRequestAttributes() + { + try + { + RequestAttributes attributes = RequestContextHolder.getRequestAttributes(); + return (ServletRequestAttributes) attributes; + } + catch (Exception e) + { + return null; + } + } + + public static Map getHeaders(HttpServletRequest request) + { + Map map = new LinkedHashMap<>(); + Enumeration enumeration = request.getHeaderNames(); + if (enumeration != null) + { + while (enumeration.hasMoreElements()) + { + String key = enumeration.nextElement(); + String value = request.getHeader(key); + map.put(key, value); + } + } + return map; + } + + public static Integer toInt(Object value, Integer defaultValue) + { + if (value == null) + { + return defaultValue; + } + if (value instanceof Integer) + { + return (Integer) value; + } + if (value instanceof Number) + { + return ((Number) value).intValue(); + } + final String valueStr = toStr(value, null); + if (StringUtils.isEmpty(valueStr)) + { + return defaultValue; + } + try + { + return Integer.parseInt(valueStr.trim()); + } + catch (Exception e) + { + return defaultValue; + } + } + + /** + * 转换为int
+ * 如果给定的值为null,或者转换失败,返回默认值null
+ * 转换失败不会报错 + * + * @param value 被转换的值 + * @return 结果 + */ + public static Integer toInt(Object value) + { + return toInt(value, null); + } + +} \ No newline at end of file diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/SqlUtil.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/SqlUtil.java new file mode 100644 index 000000000..064ccfa89 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/SqlUtil.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.framework.admin.common.utils; + +import com.webank.wedatasphere.dss.framework.admin.common.exception.AdminException; + +public class SqlUtil { + + public static String SQL_PATTERN = "[a-zA-Z0-9_\\ \\,\\.]+"; + public static String escapeOrderBySql(String value) + { + if (StringUtils.isNotEmpty(value) && !isValidOrderBySql(value)) + { + throw new AdminException("参数不符合规范,不能进行查询"); + } + return value; + } + public static boolean isValidOrderBySql(String value) + { + return value.matches(SQL_PATTERN); + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/StringUtils.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/StringUtils.java new file mode 100644 index 000000000..d7cc1d73f --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/StringUtils.java @@ -0,0 +1,428 @@ +package com.webank.wedatasphere.dss.framework.admin.common.utils; + +import org.springframework.lang.Nullable; +import org.springframework.util.AntPathMatcher; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class StringUtils extends org.apache.commons.lang3.StringUtils { + /** 空字符串 */ + private static final String NULLSTR = ""; + + /** 下划线 */ + private static final char SEPARATOR = '_'; + + /** + * 获取参数不为空值 + * + * @param value defaultValue 要判断的value + * @return value 返回值 + */ + public static T nvl(T value, T defaultValue) + { + return value != null ? value : defaultValue; + } + + /** + * * 判断一个Collection是否为空, 包含List,Set,Queue + * + * @param coll 要判断的Collection + * @return true:为空 false:非空 + */ + public static boolean isEmpty(Collection coll) + { + return isNull(coll) || coll.isEmpty(); + } + + /** + * * 判断一个Collection是否非空,包含List,Set,Queue + * + * @param coll 要判断的Collection + * @return true:非空 false:空 + */ + public static boolean isNotEmpty(Collection coll) + { + return !isEmpty(coll); + } + + /** + * * 判断一个对象数组是否为空 + * + * @param objects 要判断的对象数组 + ** @return true:为空 false:非空 + */ + public static boolean isEmpty(Object[] objects) + { + return isNull(objects) || (objects.length == 0); + } + + /** + * * 判断一个对象数组是否非空 + * + * @param objects 要判断的对象数组 + * @return true:非空 false:空 + */ + public static boolean isNotEmpty(Object[] objects) + { + return !isEmpty(objects); + } + + /** + * * 判断一个Map是否为空 + * + * @param map 要判断的Map + * @return true:为空 false:非空 + */ + public static boolean isEmpty(Map map) + { + return isNull(map) || map.isEmpty(); + } + + /** + * * 判断一个Map是否为空 + * + * @param map 要判断的Map + * @return true:非空 false:空 + */ + public static boolean isNotEmpty(Map map) + { + return !isEmpty(map); + } + + /** + * * 判断一个字符串是否为空串 + * + * @param str String + * @return true:为空 false:非空 + */ + public static boolean isEmpty(String str) + { + return isNull(str) || NULLSTR.equals(str.trim()); + } + + /** + * * 判断一个字符串是否为非空串 + * + * @param str String + * @return true:非空串 false:空串 + */ + public static boolean isNotEmpty(String str) + { + return !isEmpty(str); + } + + /** + * * 判断一个对象是否为空 + * + * @param object Object + * @return true:为空 false:非空 + */ + public static boolean isNull(Object object) + { + return object == null; + } + + /** + * * 判断一个对象是否非空 + * + * @param object Object + * @return true:非空 false:空 + */ + public static boolean isNotNull(Object object) + { + return !isNull(object); + } + + /** + * * 判断一个对象是否是数组类型(Java基本型别的数组) + * + * @param object 对象 + * @return true:是数组 false:不是数组 + */ + public static boolean isArray(Object object) + { + return isNotNull(object) && object.getClass().isArray(); + } + + /** + * 去空格 + */ + public static String trim(String str) + { + return (str == null ? "" : str.trim()); + } + + /** + * 截取字符串 + * + * @param str 字符串 + * @param start 开始 + * @return 结果 + */ + public static String substring(final String str, int start) + { + if (str == null) + { + return NULLSTR; + } + + if (start < 0) + { + start = str.length() + start; + } + + if (start < 0) + { + start = 0; + } + if (start > str.length()) + { + return NULLSTR; + } + + return str.substring(start); + } + + /** + * 截取字符串 + * + * @param str 字符串 + * @param start 开始 + * @param end 结束 + * @return 结果 + */ + public static String substring(final String str, int start, int end) + { + if (str == null) + { + return NULLSTR; + } + + if (end < 0) + { + end = str.length() + end; + } + if (start < 0) + { + start = str.length() + start; + } + + if (end > str.length()) + { + end = str.length(); + } + + if (start > end) + { + return NULLSTR; + } + + if (start < 0) + { + start = 0; + } + if (end < 0) + { + end = 0; + } + + return str.substring(start, end); + } + + /** + * 驼峰转下划线命名 + */ + public static String toUnderScoreCase(String str) + { + if (str == null) + { + return null; + } + StringBuilder sb = new StringBuilder(); + // 前置字符是否大写 + boolean preCharIsUpperCase = true; + // 当前字符是否大写 + boolean curreCharIsUpperCase = true; + // 下一字符是否大写 + boolean nexteCharIsUpperCase = true; + for (int i = 0; i < str.length(); i++) + { + char c = str.charAt(i); + if (i > 0) + { + preCharIsUpperCase = Character.isUpperCase(str.charAt(i - 1)); + } + else + { + preCharIsUpperCase = false; + } + + curreCharIsUpperCase = Character.isUpperCase(c); + + if (i < (str.length() - 1)) + { + nexteCharIsUpperCase = Character.isUpperCase(str.charAt(i + 1)); + } + + if (preCharIsUpperCase && curreCharIsUpperCase && !nexteCharIsUpperCase) + { + sb.append(SEPARATOR); + } + else if ((i != 0 && !preCharIsUpperCase) && curreCharIsUpperCase) + { + sb.append(SEPARATOR); + } + sb.append(Character.toLowerCase(c)); + } + + return sb.toString(); + } + + /** + * 是否包含字符串 + * + * @param str 验证字符串 + * @param strs 字符串组 + * @return 包含返回true + */ + public static boolean inStringIgnoreCase(String str, String... strs) + { + if (str != null && strs != null) + { + for (String s : strs) + { + if (str.equalsIgnoreCase(trim(s))) + { + return true; + } + } + } + return false; + } + + /** + * 将下划线大写方式命名的字符串转换为驼峰式。如果转换前的下划线大写方式命名的字符串为空,则返回空字符串。 例如:HELLO_WORLD->HelloWorld + * + * @param name 转换前的下划线大写方式命名的字符串 + * @return 转换后的驼峰式命名的字符串 + */ + public static String convertToCamelCase(String name) + { + StringBuilder result = new StringBuilder(); + // 快速检查 + if (name == null || name.isEmpty()) + { + // 没必要转换 + return ""; + } + else if (!name.contains("_")) + { + // 不含下划线,仅将首字母大写 + return name.substring(0, 1).toUpperCase() + name.substring(1); + } + // 用下划线将原始字符串分割 + String[] camels = name.split("_"); + for (String camel : camels) + { + // 跳过原始字符串中开头、结尾的下换线或双重下划线 + if (camel.isEmpty()) + { + continue; + } + // 首字母大写 + result.append(camel.substring(0, 1).toUpperCase()); + result.append(camel.substring(1).toLowerCase()); + } + return result.toString(); + } + + /** + * 驼峰式命名法 例如:user_name->userName + */ + public static String toCamelCase(String s) + { + if (s == null) + { + return null; + } + s = s.toLowerCase(); + StringBuilder sb = new StringBuilder(s.length()); + boolean upperCase = false; + for (int i = 0; i < s.length(); i++) + { + char c = s.charAt(i); + + if (c == SEPARATOR) + { + upperCase = true; + } + else if (upperCase) + { + sb.append(Character.toUpperCase(c)); + upperCase = false; + } + else + { + sb.append(c); + } + } + return sb.toString(); + } + + /** + * 查找指定字符串是否匹配指定字符串列表中的任意一个字符串 + * + * @param str 指定字符串 + * @param strs 需要检查的字符串数组 + * @return 是否匹配 + */ + public static boolean matches(String str, List strs) + { + if (isEmpty(str) || isEmpty(strs)) + { + return false; + } + for (String pattern : strs) + { + if (isMatch(pattern, str)) + { + return true; + } + } + return false; + } + + /** + * 判断url是否与规则配置: + * ? 表示单个字符; + * * 表示一层路径内的任意字符串,不可跨层级; + * ** 表示任意层路径; + * + * @param pattern 匹配规则 + * @param url 需要匹配的url + * @return + */ + public static boolean isMatch(String pattern, String url) + { + AntPathMatcher matcher = new AntPathMatcher(); + return matcher.match(pattern, url); + } + + + public static boolean hasText(@Nullable CharSequence str) { + return str != null && str.length() > 0 && containsText(str); + } + private static boolean containsText(CharSequence str) { + int strLen = str.length(); + + for(int i = 0; i < strLen; ++i) { + if (!Character.isWhitespace(str.charAt(i))) { + return true; + } + } + + return false; + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/TableSupport.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/TableSupport.java new file mode 100644 index 000000000..8a8bfe41c --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/common/utils/TableSupport.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.dss.framework.admin.common.utils; + +import com.webank.wedatasphere.dss.framework.admin.common.domain.PageDomain; + +public class TableSupport +{ + /** + * 当前记录起始索引 + */ + public static final String PAGE_NUM = "pageNum"; + + /** + * 每页显示记录数 + */ + public static final String PAGE_SIZE = "pageSize"; + + /** + * 排序列 + */ + public static final String ORDER_BY_COLUMN = "orderByColumn"; + + /** + * 排序的方向 "desc" 或者 "asc". + */ + public static final String IS_ASC = "isAsc"; + + /** + * 封装分页对象 + */ + public static PageDomain getPageDomain() + { + PageDomain pageDomain = new PageDomain(); + pageDomain.setPageNum(ServletUtils.getParameterToInt(PAGE_NUM)); + pageDomain.setPageSize(ServletUtils.getParameterToInt(PAGE_SIZE)); + pageDomain.setOrderByColumn(ServletUtils.getParameter(ORDER_BY_COLUMN)); + pageDomain.setIsAsc(ServletUtils.getParameter(IS_ASC)); + return pageDomain; + } + + public static PageDomain buildPageRequest() + { + return getPageDomain(); + } +} + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/conf/AdminConf.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/conf/AdminConf.java new file mode 100644 index 000000000..197b69465 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/conf/AdminConf.java @@ -0,0 +1,37 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.admin.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public interface AdminConf { + + CommonVars LDAP_ADMIN_NAME = CommonVars.apply("wds.dss.ldap.admin.name", ""); + CommonVars LDAP_ADMIN_PASS = CommonVars.apply("wds.dss.ldap.admin.password", ""); + CommonVars LDAP_URL = CommonVars.apply("wds.dss.ldap.url", ""); + CommonVars LDAP_BASE_DN = CommonVars.apply("wds.dss.ldap.base.dn", ""); + CommonVars EXCHANGE_URL = CommonVars.apply("wds.dss.exchange.url", ""); + CommonVars EXCHANGE_ADMIN_COOKIE = CommonVars.apply("wds.dss.exchange.cookie", ""); + CommonVars DS_TRUST_TOKEN = CommonVars.apply("wds.dss.trust.token", ""); + CommonVars DS_PROXY_SELF_ENABLE = CommonVars.apply("wds.dss.proxy.self.enable", true); + CommonVars DSS_PROXY_ADMIN_NAME = CommonVars.apply("wds.dss.proxy.admin.name", "adminUser"); + + String[] SUPER_ADMIN_LIST = CommonVars.apply("wds.dss.super.admin", "").getValue().split(","); + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/exception/DSSAdminErrorException.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/exception/DSSAdminErrorException.java new file mode 100644 index 000000000..833a45e26 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/exception/DSSAdminErrorException.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.framework.admin.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + + +public class DSSAdminErrorException extends DSSErrorException { + public DSSAdminErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public DSSAdminErrorException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssAdminDept.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssAdminDept.java new file mode 100644 index 000000000..5af3a5726 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssAdminDept.java @@ -0,0 +1,195 @@ +package com.webank.wedatasphere.dss.framework.admin.pojo.entity; + +import com.webank.wedatasphere.dss.framework.admin.common.domain.BaseEntity; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +public class DssAdminDept extends BaseEntity implements Serializable { + private static final long serialVersionUID = 1L; + + /** 部门ID */ + private Long id; + + /** 父部门ID */ + private Long parentId; + + /** 祖级列表 */ + private String ancestors; + + /** 部门名称 */ + private String deptName; + + /** 显示顺序 */ + private String orderNum; + + /** 负责人 */ + private String leader; + + /** 联系电话 */ + private String phone; + + /** 邮箱 */ + private String email; + + /** 部门状态:0正常,1停用 */ + private String status; + + /** 删除标志(0代表存在 2代表删除) */ + private String delFlag; + + /** 父部门名称 */ + private String parentName; + + /** 子部门 */ + private List children = new ArrayList(); + + public Long getId() + { + return id; + } + + public void setId(Long deptId) + { + this.id = deptId; + } + public void setDeptId(Long deptId) + { + this.id = deptId; + } + + public Long getParentId() + { + return parentId; + } + + public void setParentId(Long parentId) + { + this.parentId = parentId; + } + + public String getAncestors() + { + return ancestors; + } + + public void setAncestors(String ancestors) + { + this.ancestors = ancestors; + } + + + public String getDeptName() + { + return deptName; + } + + public void setDeptName(String deptName) + { + this.deptName = deptName; + } + + public String getOrderNum() + { + return orderNum; + } + + public void setOrderNum(String orderNum) + { + this.orderNum = orderNum; + } + + public String getLeader() + { + return leader; + } + + public void setLeader(String leader) + { + this.leader = leader; + } + + public String getPhone() + { + return phone; + } + + public void setPhone(String phone) + { + this.phone = phone; + } + + public String getEmail() + { + return email; + } + + public void setEmail(String email) + { + this.email = email; + } + + public String getStatus() + { + return status; + } + + public void setStatus(String status) + { + this.status = status; + } + + public String getDelFlag() + { + return delFlag; + } + + public void setDelFlag(String delFlag) + { + this.delFlag = delFlag; + } + + public String getParentName() + { + return parentName; + } + + public void setParentName(String parentName) + { + this.parentName = parentName; + } + + public List getChildren() + { + return children; + } + + public void setChildren(List children) + { + this.children = children; + } + + @Override + public String toString() { + return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE) + .append("deptId", getId()) + .append("parentId", getParentId()) + .append("ancestors", getAncestors()) + .append("deptName", getDeptName()) + .append("orderNum", getOrderNum()) + .append("leader", getLeader()) + .append("phone", getPhone()) + .append("email", getEmail()) + .append("status", getStatus()) + .append("delFlag", getDelFlag()) + .append("createBy", getCreateBy()) + .append("createTime", getCreateTime()) + .append("updateBy", getUpdateBy()) + .append("updateTime", getUpdateTime()) + .toString(); + } + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssAdminUser.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssAdminUser.java new file mode 100644 index 000000000..2dd439130 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssAdminUser.java @@ -0,0 +1,200 @@ +package com.webank.wedatasphere.dss.framework.admin.pojo.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.dss.framework.admin.common.domain.BaseEntity; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +import java.util.Date; + +/** + *

+ * 用户信息表 + *

+ */ +@TableName(value = "dss_user") +public class DssAdminUser extends BaseEntity { + private static final long serialVersionUID = 1L; + /** + * user_id + */ + @TableId(type = IdType.AUTO) + private Long id; + + /** + * 用户名 + */ + private String userName; + + /** + * 昵称 + */ + private String name; + + + private Integer isFirstLogin; + + /** + * 部门id + */ + private Long deptId; + + /** + * 是否管理员 + */ + private Integer isAdmin; + + /** + * 用户邮箱 + */ + private String email; + + /** + * 手机号码 + */ + private String phonenumber; + + /** + * 密码 + */ + private String password; + + /** + * 删除标志(0代表存在 2代表删除) + */ + private String delFlag; + + private Date lastLoginTime; + + private Integer loginNum; + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getUserName() { + return userName; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setIsFirstLogin(Integer isFirstLogin) { + this.isFirstLogin = isFirstLogin; + } + + public Integer getIsFirstLogin() { + return isFirstLogin; + } + + public void setDeptId(Long deptId) { + this.deptId = deptId; + } + + public Long getDeptId() { + return deptId; + } + + public void setIsAdmin(Integer isAdmin) { + this.isAdmin = isAdmin; + } + + public Integer getIsAdmin() { + return isAdmin; + } + + public void setEmail(String email) { + this.email = email; + } + + public String getEmail() { + return email; + } + + public void setPhonenumber(String phonenumber) { + this.phonenumber = phonenumber; + } + + public String getPhonenumber() { + return phonenumber; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getPassword() { + return password; + } + + public void setDelFlag(String delFlag) { + this.delFlag = delFlag; + } + + public String getDelFlag() { + return delFlag; + } + + public Date getLastLoginTime() { + return lastLoginTime; + } + + public void setLastLoginTime(Date lastLoginTime) { + this.lastLoginTime = lastLoginTime; + } + + public Integer getLoginNum() { + return loginNum; + } + + public void setLoginNum(Integer loginNum) { + this.loginNum = loginNum; + } + + @Override + public String toString() { + return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE) + .append("id" , getId()) + .append("userName" , getUserName()) + .append("name" , getName()) + .append("isFirstLogin" , getIsFirstLogin()) + .append("deptId" , getDeptId()) + .append("isAdmin" , getIsAdmin()) + .append("email" , getEmail()) + .append("phonenumber" , getPhonenumber()) + .append("password" , getPassword()) + .append("delFlag" , getDelFlag()) + .append("createTime" , getCreateTime()) + .append("updateTime" , getUpdateTime()) + .append("remark" , getRemark()) + .append("lastLoginTime", lastLoginTime) + .append("loginNum" , loginNum) + .toString(); + } + + private DssAdminDept dept; + + public DssAdminDept getDept() { + return dept; + } + + public void setDept(DssAdminDept dept) { + this.dept = dept; + } + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangeTask.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangeTask.java new file mode 100644 index 000000000..791c01613 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangeTask.java @@ -0,0 +1,64 @@ +package com.webank.wedatasphere.dss.framework.admin.pojo.entity; + +/** + * @Auther: Han Tang + * @Date: 2022/1/18-01-18-15:58 + */ +public class DssExchangeTask { + private int id; + private String jobName; + private String jobCorn; + private String jobDesc; + private String createTime; + private String jobStatus; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getJobCorn() { + return jobCorn; + } + + public void setJobCorn(String jobCorn) { + this.jobCorn = jobCorn; + } + + public String getJobDesc() { + return jobDesc; + } + + public void setJobDesc(String jobDesc) { + this.jobDesc = jobDesc; + } + + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } + + public String getJobStatus() { + return jobStatus; + } + + public void setJobStatus(String jobStatus) { + this.jobStatus = jobStatus; + } + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangeTaskRes.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangeTaskRes.java new file mode 100644 index 000000000..781a63bdc --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangeTaskRes.java @@ -0,0 +1,57 @@ +package com.webank.wedatasphere.dss.framework.admin.pojo.entity; + +import java.util.List; + +/** + * @Auther: Han Tang + * @Date: 2022/1/18-01-18-18:06 + */ +public class DssExchangeTaskRes { + + private int page; + private int totalItems; + private int totalPages; + private int pageSize; + private List dssExchangeTaskList; + public int getPageSize() { + return pageSize; + } + + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } + + public int getPage() { + return page; + } + + public void setPage(int page) { + this.page = page; + } + + public int getTotalItems() { + return totalItems; + } + + public void setTotalItems(int totalItems) { + this.totalItems = totalItems; + } + + public int getTotalPages() { + return totalPages; + } + + public void setTotalPages(int totalPages) { + this.totalPages = totalPages; + } + + public List getDssExchangeTaskList() { + return dssExchangeTaskList; + } + + public void setDssExchangeTaskList(List dssExchangeTaskList) { + this.dssExchangeTaskList = dssExchangeTaskList; + } + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangisProject.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangisProject.java new file mode 100644 index 000000000..84929b925 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssExchangisProject.java @@ -0,0 +1,93 @@ +package com.webank.wedatasphere.dss.framework.admin.pojo.entity; + +import java.util.List; + +/** + * @Auther: Han Tang + * @Date: 2022/1/18-01-18-14:05 + */ +public class DssExchangisProject { + private int id; + private String projectName; + private int parentId; + private String createUser; + private String createTime; + private String modifyUser; + private String modifyTime; + private int level; + private List children; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public int getParentId() { + return parentId; + } + + public void setParentId(int parentId) { + this.parentId = parentId; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } + + public String getModifyUser() { + return modifyUser; + } + + public void setModifyUser(String modifyUser) { + this.modifyUser = modifyUser; + } + + public String getModifyTime() { + return modifyTime; + } + + public void setModifyTime(String modifyTime) { + this.modifyTime = modifyTime; + } + + public int getLevel() { + return level; + } + + public void setLevel(int level) { + this.level = level; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssProxyUser.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssProxyUser.java new file mode 100644 index 000000000..d531af351 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssProxyUser.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.dss.framework.admin.pojo.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.dss.framework.admin.common.domain.BaseEntity; + + +@TableName(value = "dss_proxy_user") +public class DssProxyUser extends BaseEntity { + private static final long serialVersionUID = 1L; + + @TableId(type = IdType.AUTO) + private Long id; + private String userName; + /**代理用户名*/ + private String proxyUserName; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getProxyUserName() { + return proxyUserName; + } + + public void setProxyUserName(String proxyUserName) { + this.proxyUserName = proxyUserName; + } + + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssScriptDownloadAudit.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssScriptDownloadAudit.java new file mode 100644 index 000000000..91e086ed0 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/DssScriptDownloadAudit.java @@ -0,0 +1,86 @@ +package com.webank.wedatasphere.dss.framework.admin.pojo.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import org.springframework.format.annotation.DateTimeFormat; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Date; + +/** + * @Auther: Han Tang + * @Date: 2022/1/10-01-10-15:48 + */ + + + +@TableName(value = "dss_workspace_download_audit") +public class DssScriptDownloadAudit { + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + private String creator; + private String tenant; + private String path; + @TableField("`sql`") + @NotEmpty(message = "查询语句不能为空") + private String sql; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getTenant() { + return tenant; + } + + public void setTenant(String tenant) { + this.tenant = tenant; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/TreeSelect.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/TreeSelect.java new file mode 100644 index 000000000..9e4a1829b --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/pojo/entity/TreeSelect.java @@ -0,0 +1,66 @@ +package com.webank.wedatasphere.dss.framework.admin.pojo.entity; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.io.Serializable; +import java.util.List; +import java.util.stream.Collectors; + +public class TreeSelect implements Serializable +{ + private static final long serialVersionUID = 1L; + + /** 节点ID */ + private Long id; + + /** 节点名称 */ + private String label; + + /** 子节点 */ + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public List children; + + public TreeSelect() + { + + } + + public TreeSelect(DssAdminDept dept) + { + this.id = dept.getId(); + this.label = dept.getDeptName(); + this.children = dept.getChildren().stream().map(TreeSelect::new).collect(Collectors.toList()); + } + + public Long getId() + { + return id; + } + + public void setId(Long id) + { + this.id = id; + } + + public String getLabel() + { + return label; + } + + public void setLabel(String label) + { + this.label = label; + } + + public List getChildren() + { + return children; + } + + public void setChildren(List children) + { + this.children = children; + } + + +} \ No newline at end of file diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/BaseController.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/BaseController.java new file mode 100644 index 000000000..be53326e4 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/BaseController.java @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.dss.framework.admin.restful; + +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.framework.admin.common.domain.Message; +import com.webank.wedatasphere.dss.framework.admin.common.domain.PageDomain; +import com.webank.wedatasphere.dss.framework.admin.common.domain.ResponseEnum; +import com.webank.wedatasphere.dss.framework.admin.common.domain.TableDataInfo; +import com.webank.wedatasphere.dss.framework.admin.common.utils.SqlUtil; +import com.webank.wedatasphere.dss.framework.admin.common.utils.StringUtils; +import com.webank.wedatasphere.dss.framework.admin.common.utils.TableSupport; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class BaseController { + + + /** + * 设置请求分页数据 + */ + protected void startPage() + { + PageDomain pageDomain = TableSupport.buildPageRequest(); + Integer pageNum = pageDomain.getPageNum(); + Integer pageSize = pageDomain.getPageSize(); + if (StringUtils.isNotNull(pageNum) && StringUtils.isNotNull(pageSize)) + { + String orderBy = SqlUtil.escapeOrderBySql(pageDomain.getOrderBy()); + PageHelper.startPage(pageNum, pageSize, orderBy); + } + } + + /** + * 响应请求分页数据 + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + protected TableDataInfo getDataTable(List list) + { + TableDataInfo rspData = new TableDataInfo(); + rspData.setStatus(ResponseEnum.SUCCESS.getStatus()); + HashMap data = (HashMap) Message.ok().data("total" , new PageInfo(list).getTotal()).data("userList" , list).getData(); + rspData.setData(data); + rspData.setMessage("查询成功"); + + return rspData; + } + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssAuditController.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssAuditController.java new file mode 100644 index 000000000..861024285 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssAuditController.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.dss.framework.admin.restful; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssScriptDownloadAudit; +import com.webank.wedatasphere.dss.framework.admin.service.DssScriptDownloadService; +import lombok.extern.slf4j.Slf4j; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import org.apache.linkis.server.Message; + +import javax.servlet.http.HttpServletRequest; +import javax.validation.Valid; +import java.util.List; + +/** + * @Auther: Han Tang + * @Date: 2022/1/11-01-11-14:20 + */ + + +@RestController +@RequestMapping(path = "/dss/framework/admin/audit", produces = {"application/json"}) +@Slf4j +public class DssAuditController { + + private static final Logger LOGGER = LoggerFactory.getLogger(DssAuditController.class); + @Autowired + DssScriptDownloadService dssScriptDownloadService; + + @RequestMapping(path = "script/download/save", method = RequestMethod.POST) + public Message saveScriptDownload(@RequestBody @Valid DssScriptDownloadAudit dssScriptDownloadAudit, HttpServletRequest request) { + String userName = SecurityFilter.getLoginUsername(request); +// String userName = "demo"; + dssScriptDownloadAudit.setCreator(userName); + String sql = dssScriptDownloadAudit.getSql(); + if(sql.length()>2000){ + sql = sql.substring(0,2000); + dssScriptDownloadAudit.setSql(sql); + } + dssScriptDownloadService.save(dssScriptDownloadAudit); + Message message = Message.ok(); + return message; + } + + + @RequestMapping(path = "script/download/query", method = RequestMethod.GET) + public Message getScriptDownload(@RequestParam(required = false) String userName,@RequestParam(required = false) String startTime,@RequestParam(required = false) String endTime,@RequestParam Integer pn) { + PageHelper.startPage(pn, 10, true); + List userPage = dssScriptDownloadService.getDownloadAuditList(userName,startTime,endTime); + PageInfo pageInfo = new PageInfo<>(userPage); + Message message = Message.ok().data("data", pageInfo); + return message; + } +} + + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssFrameworkAdminDeptController.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssFrameworkAdminDeptController.java new file mode 100644 index 000000000..9c0334239 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssFrameworkAdminDeptController.java @@ -0,0 +1,100 @@ +package com.webank.wedatasphere.dss.framework.admin.restful; + + +import com.webank.wedatasphere.dss.framework.admin.common.constant.UserConstants; +import com.webank.wedatasphere.dss.framework.admin.common.domain.Message; +import com.webank.wedatasphere.dss.framework.admin.common.utils.StringUtils; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminDept; +import com.webank.wedatasphere.dss.framework.admin.service.DssAdminDeptService; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import java.util.List; + +@RequestMapping(path = "/dss/framework/admin/dept", produces = {"application/json"}) +@RestController +public class DssFrameworkAdminDeptController { + @Resource + private DssAdminDeptService dssAdminDeptService; + + + @RequestMapping(path = "list", method = RequestMethod.GET) + public Message listAll(@RequestParam(value = "parentId", required = false) Long parentId, @RequestParam(value = "deptName", required = false) String deptName) { + + DssAdminDept dept = new DssAdminDept(); + dept.setParentId(parentId); + dept.setDeptName(deptName); + List list = dssAdminDeptService.selectDeptList(dept); + return Message.ok().data("deptList", list).message("成功"); + } + + @RequestMapping(method = RequestMethod.POST) + public Message add(@RequestBody DssAdminDept dssAdminDept) { + if (UserConstants.NOT_UNIQUE.equals(dssAdminDeptService.checkDeptNameUnique(dssAdminDept))) { + return Message.error().message("新增部门'" + dssAdminDept.getDeptName() + "'失败,部门名称已存在"); + } else if (dssAdminDept.getDeptName().contains(UserConstants.SINGLE_SPACE)) { + return Message.error().message("新增部门'" + dssAdminDept.getDeptName() + "'部门名称中不能含有空格"); + } else if (dssAdminDeptService.checkDeptFinalStage(dssAdminDept.getParentId())) { + return Message.error().message("新增部门'" + dssAdminDept.getDeptName() + "'失败,该部门是末级部门,不能新增下级部门"); + } + + int saveResult = dssAdminDeptService.insertDept(dssAdminDept); + System.out.println(saveResult); + if (saveResult >= 1) { + return Message.ok().message("保存成功"); + } else { + return Message.error().message("保存失败"); + } + } + + /** + * 获取部门下拉树列表 + */ + @RequestMapping(path = "treeselect", method = RequestMethod.GET) + public Message treeselect(DssAdminDept dept) { + List depts = dssAdminDeptService.selectDeptList(dept); + return Message.ok().data("deptTree", dssAdminDeptService.buildDeptTreeSelect(depts)).message("树形部门获取成功"); + } + + + @RequestMapping(path = "{deptId}", method = RequestMethod.GET) + public Message getInfo(@PathVariable("deptId") Long deptId) { + return Message.ok().data("deptInfo", dssAdminDeptService.selectDeptById(deptId)); + } + + + @RequestMapping(path = "edit", method = RequestMethod.POST) + public Message edit(@Validated @RequestBody DssAdminDept dept) { + if (UserConstants.NOT_UNIQUE.equals(dssAdminDeptService.checkDeptNameUnique(dept))) { + return Message.error().message("修改部门'" + dept.getDeptName() + "'失败,部门名称已存在"); + } else if (dept.getDeptName().contains(UserConstants.SINGLE_SPACE)) { + return Message.error().message("修改部门'" + dept.getDeptName() + "'部门名称中不能含有空格"); + } else if (dept.getParentId().equals(dept.getId())) { + return Message.error().message("修改部门'" + dept.getDeptName() + "'失败,上级部门不能是自己"); + } else if (StringUtils.equals(UserConstants.DEPT_DISABLE, dept.getStatus()) + && dssAdminDeptService.selectNormalChildrenDeptById(dept.getId()) > 0) { + return Message.error().message("该部门包含未停用的子部门!"); + } + + return Message.ok().data("修改成功", dssAdminDeptService.updateDept(dept)); + } + + /** + * 删除部门 + */ + + @RequestMapping(path = "{deptId}", method = RequestMethod.POST) + public Message remove(@PathVariable("deptId") Long deptId) { + if (dssAdminDeptService.hasChildById(deptId)) { + return Message.error().message("存在下级部门,不允许删除"); + } + if (dssAdminDeptService.checkDeptExistUser(deptId)) { + return Message.error().message("部门存在用户,不允许删除"); + } + return Message.ok().data("删除成功", dssAdminDeptService.deleteDeptById(deptId)); + } + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssFrameworkAdminUserController.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssFrameworkAdminUserController.java new file mode 100644 index 000000000..2690e11d4 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssFrameworkAdminUserController.java @@ -0,0 +1,128 @@ +package com.webank.wedatasphere.dss.framework.admin.restful; + + +import com.webank.wedatasphere.dss.framework.admin.common.constant.UserConstants; +import com.webank.wedatasphere.dss.framework.admin.common.domain.Message; +import com.webank.wedatasphere.dss.framework.admin.common.domain.PasswordResult; +import com.webank.wedatasphere.dss.framework.admin.common.domain.TableDataInfo; +import com.webank.wedatasphere.dss.framework.admin.common.utils.PasswordUtils; +import com.webank.wedatasphere.dss.framework.admin.common.utils.StringUtils; +import com.webank.wedatasphere.dss.framework.admin.conf.AdminConf; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminUser; +import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; +import com.webank.wedatasphere.dss.framework.admin.service.LdapService; +import com.webank.wedatasphere.dss.framework.admin.xml.DssUserMapper; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardWarnException; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.linkis.server.security.SecurityFilter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RequestMapping(path = "/dss/framework/admin/user", produces = {"application/json"}) +@RestController +public class DssFrameworkAdminUserController extends BaseController { + @Resource + private DssAdminUserService dssAdminUserService; + @Autowired + private LdapService ldapService; + @Autowired + DssUserMapper dssUserMapper; + + @RequestMapping(path = "list", method = RequestMethod.GET) +// public TableDataInfo list(DssAdminUser user) { + public TableDataInfo list(@RequestParam(value = "userName", required = false) String userName, + @RequestParam(value = "deptId", required = false) Long deptId, + @RequestParam(value = "phonenumber", required = false) String phonenumber, + @RequestParam(value = "beginTime", required = false) String beginTime, + @RequestParam(value = "endTime", required = false) String endTime) { + DssAdminUser user = new DssAdminUser(); + user.setUserName(userName); + user.setDeptId(deptId); + user.setPhonenumber(phonenumber); + Map params = new HashMap<>(); + params.put("beginTime", beginTime); + params.put("endTime", endTime); + user.setParams(params); + startPage(); + List userList = dssAdminUserService.selectUserList(user); + return getDataTable(userList); + } + + @RequestMapping(path = "add", method = RequestMethod.POST) + public Message add(@Validated @RequestBody DssAdminUser user, HttpServletRequest req) { + try { + PasswordResult passwordResult = PasswordUtils.checkPwd(user.getPassword(), user); + if (UserConstants.NOT_UNIQUE.equals(dssAdminUserService.checkUserNameUnique(user.getUserName()))) { + return Message.error().message("新增用户'" + user.getUserName() + "'失败,登录账号已存在"); + } else if (user.getUserName().contains(UserConstants.SINGLE_SPACE)) { + return Message.error().message("新增用户'" + user.getUserName() + "'用户名中不能含有空格"); + } else if (StringUtils.isNotEmpty(user.getPhonenumber()) + && UserConstants.NOT_UNIQUE.equals(dssAdminUserService.checkPhoneUnique(user))) { + return Message.error().message("新增用户'" + user.getUserName() + "'失败,手机号码已存在"); + } else if (StringUtils.isNotEmpty(user.getEmail()) + && UserConstants.NOT_UNIQUE.equals(dssAdminUserService.checkEmailUnique(user))) { + return Message.error().message("新增用户'" + user.getUserName() + "'失败,邮箱账号已存在"); + } else if (!PasswordResult.PASSWORD_RULE_PASS.equals(passwordResult)) { + return Message.error().data("弱密码请关注:", passwordResult.getMessage()); + } + boolean ldapExist = ldapService.exist(AdminConf.LDAP_ADMIN_NAME.getValue(), AdminConf.LDAP_ADMIN_PASS.getValue(), AdminConf.LDAP_URL.getValue(), AdminConf.LDAP_BASE_DN.getValue(), user.getUserName()); + if (ldapExist) { + return Message.error().message("新增用户'" + user.getUserName() + "'失败,登录账号在ldap已存在"); + } + String pwd = user.getPassword(); + user.setPassword(DigestUtils.md5Hex(pwd)); + user.setCreateBy(SecurityFilter.getLoginUsername(req)); + int rows = dssAdminUserService.insertUser(user, getWorkspace(req)); + String userName = user.getUserName(); + ldapService.addUser(AdminConf.LDAP_ADMIN_NAME.getValue(), AdminConf.LDAP_ADMIN_PASS.getValue(), AdminConf.LDAP_URL.getValue(), AdminConf.LDAP_BASE_DN.getValue(), userName, pwd); + return Message.ok().data("rows", rows).message("新增成功"); + } catch (Exception exception) { + return Message.error().data("rows", 0).message(exception.getMessage()); + } + + } + + private Workspace getWorkspace(HttpServletRequest req) { + Workspace workspace = new Workspace(); + try { + SSOHelper.addWorkspaceInfo(req, workspace); + } catch (AppStandardWarnException ignored) {} // ignore it. + return workspace; + } + + + @RequestMapping(path = "{id}", method = RequestMethod.GET) + public Message getInfo(@PathVariable("id") Long userId) { + return Message.ok().data("users", dssAdminUserService.selectUserById(userId)); + } + + @RequestMapping(path = "userInfo", method = RequestMethod.GET) + public Message getLoginUserInfo(HttpServletRequest request) { + String userName = SecurityFilter.getLoginUsername(request); + return Message.ok().data("userInfo", dssAdminUserService.selectUserByName(userName)); + } + + + @RequestMapping(path = "edit", method = RequestMethod.POST) + public Message edit(@Validated @RequestBody DssAdminUser user, HttpServletRequest req) { + if (StringUtils.isNotEmpty(user.getPhonenumber()) + && UserConstants.NOT_UNIQUE.equals(dssAdminUserService.checkPhoneUnique(user))) { + return Message.error().message("修改用户'" + user.getUserName() + "'失败,手机号码已存在"); + } else if (StringUtils.isNotEmpty(user.getEmail()) + && UserConstants.NOT_UNIQUE.equals(dssAdminUserService.checkEmailUnique(user))) { + return Message.error().message("修改用户'" + user.getUserName() + "'失败,邮箱账号已存在"); + } + return Message.ok().data("修改用户成功。", dssAdminUserService.updateUser(user, getWorkspace(req))); + } + +} + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssProxyUserController.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssProxyUserController.java new file mode 100644 index 000000000..822ee5c08 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/restful/DssProxyUserController.java @@ -0,0 +1,115 @@ +package com.webank.wedatasphere.dss.framework.admin.restful; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.framework.admin.common.utils.StringUtils; +import com.webank.wedatasphere.dss.framework.admin.exception.DSSAdminErrorException; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssProxyUser; +import com.webank.wedatasphere.dss.framework.admin.service.DssProxyUserService; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.conf.ServerConfiguration; +import org.apache.linkis.server.security.ProxyUserSSOUtils; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import scala.Tuple2; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.framework.admin.conf.AdminConf.*; + +@RequestMapping(path = "/dss/framework/admin/user", produces = {"application/json"}) +@RestController +public class DssProxyUserController { + private Boolean sslEnable = (Boolean) ServerConfiguration.BDP_SERVER_SECURITY_SSL().getValue(); + private String PROXY_USER_TICKET_ID_STRING = ServerConfiguration.LINKIS_SERVER_SESSION_PROXY_TICKETID_KEY().getValue(); + protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); + @Autowired + DssProxyUserService dssProxyUserService; + + @RequestMapping(path = "proxy/list", method = RequestMethod.GET) + public Message getProxyUserList(HttpServletRequest request) { + String username = SecurityFilter.getLoginUsername(request); + + List userList = dssProxyUserService.selectProxyUserList(username); + List proxyUserNameList=userList.stream().map(dssProxyUser -> dssProxyUser.getProxyUserName()).collect(Collectors.toList()); + if(DS_PROXY_SELF_ENABLE.getValue()) { + proxyUserNameList.add(username); + } + return Message.ok().data("proxyUserList", proxyUserNameList); + } + + @RequestMapping(path = "proxy/addUserCookie", method = RequestMethod.POST) + public Message setProxyUserCookie(@RequestBody DssProxyUser userRep, HttpServletRequest req, HttpServletResponse resp) { + + String username = SecurityFilter.getLoginUsername(req); + String trustCode = DS_TRUST_TOKEN.getValue(); + try { + if (userRep.getUserName().equals(username)) { + if (StringUtils.isEmpty(userRep.getUserName())) { + DSSExceptionUtils.dealErrorException(100101, "User name is empty", DSSAdminErrorException.class); + } else if (StringUtils.isEmpty(userRep.getProxyUserName())) { + DSSExceptionUtils.dealErrorException(100102, "Proxy user name is empty", DSSAdminErrorException.class); + } else if (dssProxyUserService.isExists(userRep.getUserName(), userRep.getProxyUserName())) { + for (Cookie cookie : req.getCookies()) { + if (null != cookie && cookie.getName().equalsIgnoreCase(PROXY_USER_TICKET_ID_STRING)) { + cookie.setValue(null); + cookie.setMaxAge(0); + resp.addCookie(cookie); + } + } + } + Tuple2 userTicketIdKv = ProxyUserSSOUtils.getProxyUserTicketKV(userRep.getProxyUserName(), trustCode); + Cookie cookie = new Cookie(userTicketIdKv._1, userTicketIdKv._2); + cookie.setMaxAge(-1); + if (sslEnable){ + cookie.setSecure(true); + } + cookie.setPath("/"); + resp.addCookie(cookie); + + } else { + DSSExceptionUtils.dealErrorException(100103,"The requested user name is not a login user",DSSAdminErrorException.class); + } + return Message.ok("Success to add proxy user into cookie"); + + } catch (Exception exception) { + LOGGER.error("Failed to set cookie for proxy user", exception); + return Message.error(ExceptionUtils.getRootCauseMessage(exception)); + } + + } + + @RequestMapping(path = "proxy/add", method = RequestMethod.POST) + public Message add(@RequestBody DssProxyUser userRep, HttpServletRequest req) { + String username = SecurityFilter.getLoginUsername(req); + + try { + if(!username.equals(DSS_PROXY_ADMIN_NAME.getValue())){ + DSSExceptionUtils.dealErrorException(100104, "Only administrators can add proxy users", DSSAdminErrorException.class); + } + if(StringUtils.isEmpty(userRep.getUserName())){ + DSSExceptionUtils.dealErrorException(100105, "User name is empty", DSSAdminErrorException.class); + }else if(StringUtils.isEmpty(userRep.getProxyUserName())){ + DSSExceptionUtils.dealErrorException(100106, "Proxy user name is empty", DSSAdminErrorException.class); + }else if (dssProxyUserService.isExists(userRep.getUserName(),userRep.getProxyUserName())) { + DSSExceptionUtils.dealErrorException(100107, "Failed to add proxy user,'userName:" + userRep.getUserName() + ",proxyName:"+userRep.getProxyUserName()+" already exists", DSSAdminErrorException.class); + } + dssProxyUserService.insertProxyUser(userRep); + return Message.ok("Success to add proxy user"); + } catch (Exception exception) { + LOGGER.error("Failed to add proxy user", exception); + return Message.error(ExceptionUtils.getRootCauseMessage(exception)); + } + + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssAdminDeptService.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssAdminDeptService.java new file mode 100644 index 000000000..3766c9ea0 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssAdminDeptService.java @@ -0,0 +1,102 @@ +package com.webank.wedatasphere.dss.framework.admin.service; + +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminDept; +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.TreeSelect; + +import java.util.List; +import java.util.Map; + +public interface DssAdminDeptService extends IService { + + /** + * 查询部门管理数据 + * + * @param dept 部门信息 + * @return 部门信息集合 + */ + public List selectDeptList(DssAdminDept dept); + + /** + * 构建前端所需要树结构 + * + * @param depts 部门列表 + * @return 树结构列表 + */ + public List buildDeptTree(List depts); + + /** + * 构建前端所需要下拉树结构 + * + * @param depts 部门列表 + * @return 下拉树结构列表 + */ + public List buildDeptTreeSelect(List depts); + + + /** + * 根据部门ID查询信息 + * + * @param deptId 部门ID + * @return 部门信息 + */ + public DssAdminDept selectDeptById(Long deptId); + + /** + * 根据ID查询所有子部门(正常状态) + * + * @param deptId 部门ID + * @return 子部门数 + */ + public int selectNormalChildrenDeptById(Long deptId); + + /** + * 是否存在部门子节点 + * + * @param deptId 部门ID + * @return 结果 + */ + public boolean hasChildById(Long deptId); + + /** + * 查询部门是否存在用户 + * + * @param deptId 部门ID + * @return 结果 true 存在 false 不存在 + */ + public boolean checkDeptExistUser(Long deptId); + + /** + * 校验部门名称是否唯一 + * + * @param dept 部门信息 + * @return 结果 + */ + public String checkDeptNameUnique(DssAdminDept dept); + + /** + * 新增保存部门信息 + * + * @param dept 部门信息 + * @return 结果 + */ + public int insertDept(DssAdminDept dept); + + /** + * 修改保存部门信息 + * + * @param dept 部门信息 + * @return 结果 + */ + public int updateDept(DssAdminDept dept); + + /** + * 删除部门管理信息 + * + * @param id 部门ID + * @return 结果 + */ + public int deleteDeptById(Long id); + + boolean checkDeptFinalStage(Long parentId); +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssAdminUserService.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssAdminUserService.java new file mode 100644 index 000000000..dd62f452d --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssAdminUserService.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.framework.admin.service; + +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminUser; +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; + +public interface DssAdminUserService extends IService { + + String checkUserNameUnique(String username); + + String checkPhoneUnique(DssAdminUser user); + + String checkEmailUnique(DssAdminUser user); + + void insertOrUpdateUser(String username, Workspace workspace); + + int insertUser(DssAdminUser user, Workspace workspace); + + List selectUserList(DssAdminUser user); + + DssAdminUser selectUserById(Long userId); + + DssAdminUser selectUserByName(String username); + + int updateUser(DssAdminUser user, Workspace workspace); + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssExchangeService.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssExchangeService.java new file mode 100644 index 000000000..eb7acae49 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssExchangeService.java @@ -0,0 +1,115 @@ +package com.webank.wedatasphere.dss.framework.admin.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; +import com.webank.wedatasphere.dss.framework.admin.conf.AdminConf; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssExchangeTask; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssExchangeTaskRes; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssExchangisProject; + +import com.webank.wedatasphere.dss.framework.admin.util.OkHttpHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +/** + * @Auther: Han Tang + * @Date: 2022/1/18-01-18-15:10 + */ +@Service +public class DssExchangeService { + + private static final String PROJECT_TREE_PATH = "/api/v1/project/tree"; + private static final String TASK_TREE_PATH = "/api/v1/jobinfo/pageList"; + private static final Logger LOGGER = LoggerFactory.getLogger(OkHttpHelper.class); + + public List queryExchangeProject(String userName) throws Exception { + String url = AdminConf.EXCHANGE_URL.getValue() + PROJECT_TREE_PATH + "/"+userName; + Request getRequest = new Request.Builder() + .url(url) + .addHeader("Content-Type", "application/json") + .addHeader("Cookie", AdminConf.EXCHANGE_ADMIN_COOKIE.getValue()) + .build(); + Response response = OkHttpHelper.syncGet(getRequest); + JsonObject returnData = new JsonParser().parse(response.body().string()).getAsJsonObject(); + LOGGER.info(returnData.toString()); + JsonArray jsonArray = returnData.getAsJsonArray("data"); + List projectList = new ArrayList<>(); + List dssExchangisProjects = getExchangeProjectList(projectList, jsonArray); + return dssExchangisProjects; + } + + public DssExchangeTaskRes queryExchangeTask(int projectId, String userName, int pageNum, String fullName) throws Exception { + String url = AdminConf.EXCHANGE_URL.getValue() + TASK_TREE_PATH + "/" + userName + + "?projectId=" + projectId + "&page=" + pageNum + "&pageSize=10&fuzzyName=&jobId="; + Request getRequest = new Request.Builder() + .url(url) + .addHeader("Content-Type", "application/json") + .addHeader("Cookie", AdminConf.EXCHANGE_ADMIN_COOKIE.getValue()) + .get() + .build(); + Response response = OkHttpHelper.syncGet(getRequest); + JsonObject returnData = new JsonParser().parse(response.body().string()).getAsJsonObject(); + LOGGER.info(returnData.toString()); + DssExchangeTaskRes dssExchangeTaskRes = new DssExchangeTaskRes(); + JsonObject resJsonObject = returnData.getAsJsonObject("data"); + JsonArray resJsonArray = null; + if (resJsonObject != null) { + resJsonArray = resJsonObject.getAsJsonArray("data"); + } + + dssExchangeTaskRes.setPage(resJsonObject.get("page").getAsInt()); + dssExchangeTaskRes.setPageSize(resJsonObject.get("pageSize").getAsInt()); + dssExchangeTaskRes.setTotalPages(resJsonObject.get("totalPages").getAsInt()); + dssExchangeTaskRes.setTotalItems(resJsonObject.get("totalItems").getAsInt()); + List taskList = new ArrayList<>(); + if (resJsonArray != null) { + for (JsonElement jsonElement : resJsonArray) { + JsonObject jsonObject = jsonElement.getAsJsonObject(); + DssExchangeTask dssExchangeTask = new DssExchangeTask(); + dssExchangeTask.setId(jsonObject.get("id").getAsInt()); + dssExchangeTask.setJobName(jsonObject.get("jobName") == null ? null : jsonObject.get("jobName").getAsString()); + dssExchangeTask.setJobCorn(jsonObject.get("jobCorn") == null ? null : jsonObject.get("jobCorn").getAsString()); + dssExchangeTask.setJobDesc(jsonObject.get("jobDesc") == null ? null : jsonObject.get("jobDesc").getAsString()); + dssExchangeTask.setCreateTime(jsonObject.get("createTime") == null ? null : jsonObject.get("createTime").getAsString()); + dssExchangeTask.setJobStatus(jsonObject.get("jobStatus") == null ? null : jsonObject.get("jobStatus").getAsString()); + taskList.add(dssExchangeTask); + } + } + + dssExchangeTaskRes.setDssExchangeTaskList(taskList); + return dssExchangeTaskRes; + } + + + public String getSellScript(int taskId, int projectId) { + String shellScript = "str=`curl -X GET --data '{\"project_id\":" + projectId + ",\"task_id\":" + taskId + "}' " + + "--header 'Content-Type: application/json' --header 'Accept: application/json' " + + "--header 'Cookie:" + AdminConf.EXCHANGE_ADMIN_COOKIE.getValue() + "' " + AdminConf.EXCHANGE_URL.getValue() + + "/api/v1/jobinfo/runTask/" + taskId + "?userName=admin`;if [[ ${str} =~ 'job execution successed' ]];then exit 0;else exit 1;fi"; + return shellScript; + } + + public List getExchangeProjectList(List projectList, JsonArray jsonArray) { + if (jsonArray != null) { + for (JsonElement jsonElement : jsonArray) { + DssExchangisProject dssExchangisProject = new DssExchangisProject(); + dssExchangisProject.setProjectName(jsonElement.getAsJsonObject().get("projectName") == null ? null : jsonElement.getAsJsonObject().get("projectName").getAsString()); + dssExchangisProject.setId(jsonElement.getAsJsonObject().get("id") == null ? null : jsonElement.getAsJsonObject().get("id").getAsInt()); + projectList.add(dssExchangisProject); + JsonArray childJsonArray = jsonElement.getAsJsonObject().getAsJsonArray("children"); + if (childJsonArray != null && childJsonArray.size() > 0) { + getExchangeProjectList(projectList, childJsonArray); + } + } + } + return projectList; + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssProxyUserService.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssProxyUserService.java new file mode 100644 index 000000000..5442a01bd --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssProxyUserService.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.framework.admin.service; + + +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssProxyUser; + +import java.util.List; + +public interface DssProxyUserService { + + /** + * 查询代理用户数据 + * + * @param userName 查询代理用户的用户名 + * @return 代理用户的集合 + */ + List selectProxyUserList(String userName); + + List getProxyUserNameList(String userName); + + int insertProxyUser(DssProxyUser dssProxyUser); + + boolean isExists(String userName,String proxyUserName); + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssScriptDownloadService.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssScriptDownloadService.java new file mode 100644 index 000000000..26ba26791 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/DssScriptDownloadService.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.framework.admin.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssScriptDownloadAudit; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @Auther: Han Tang + * @Date: 2022/1/11-01-11-15:11 + */ +public interface DssScriptDownloadService extends IService { + + List getDownloadAuditList( String userName, String startIme, String endTime); + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/LdapService.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/LdapService.java new file mode 100644 index 000000000..e1dbb4266 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/LdapService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.framework.admin.service; + +import javax.naming.NamingException; + +public interface LdapService { + + void addUser(String adminName, String adminPassword, String ldapUrl, String baseDN, String userName, String pwd) throws NamingException; + + void update(String adminName, String adminPassword, String ldapUrl, String baseDN, String userName, String pwd) throws NamingException; + + boolean exist(String adminName, String adminPassword, String ldapUrl, String baseDN, String userName) throws NamingException; +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssAdminDeptServiceImpl.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssAdminDeptServiceImpl.java new file mode 100644 index 000000000..e653d459c --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssAdminDeptServiceImpl.java @@ -0,0 +1,248 @@ +package com.webank.wedatasphere.dss.framework.admin.service.impl; + +import com.webank.wedatasphere.dss.framework.admin.common.constant.UserConstants; +import com.webank.wedatasphere.dss.framework.admin.common.exception.AdminException; +import com.webank.wedatasphere.dss.framework.admin.common.utils.StringUtils; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminDept; +import com.webank.wedatasphere.dss.framework.admin.xml.DssAdminDeptMapper; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.TreeSelect; +import com.webank.wedatasphere.dss.framework.admin.service.DssAdminDeptService; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class DssAdminDeptServiceImpl extends ServiceImpl implements DssAdminDeptService { + @Resource + private DssAdminDeptMapper dssAdminDeptMapper; + + /** + * 校验部门名称是否唯一 + * + * @param dssAdminDept 部门信息 + * @return 结果 + */ + @Override + public String checkDeptNameUnique(DssAdminDept dssAdminDept) { + Long deptId = StringUtils.isNull(dssAdminDept.getId()) ? -1L : dssAdminDept.getId(); + DssAdminDept info = dssAdminDeptMapper.checkDeptNameUnique(dssAdminDept.getDeptName(), dssAdminDept.getParentId()); + if (StringUtils.isNotNull(info) && info.getId().longValue() != deptId.longValue()) { + return UserConstants.NOT_UNIQUE; + } + return UserConstants.UNIQUE; + } + + /** + * 新增保存部门信息 + * + * @param dept 部门信息 + * @return 结果 + */ + @Override + public int insertDept(DssAdminDept dept) { + DssAdminDept info = dssAdminDeptMapper.selectDeptById(dept.getParentId()); + // 如果父节点不为正常状态,则不允许新增子节点 + if (!UserConstants.DEPT_NORMAL.equals(info.getStatus())) { + throw new AdminException("部门停用,不允许新增"); + } + dept.setAncestors(info.getAncestors() + "," + dept.getParentId()); + return dssAdminDeptMapper.insertDept(dept); + } + + + @Override + public List selectDeptList(DssAdminDept dept) { + return dssAdminDeptMapper.selectDeptList(dept); + } + + + /** + * 构建前端所需要下拉树结构 + * + * @param depts 部门列表 + * @return 下拉树结构列表 + */ + @Override + public List buildDeptTreeSelect(List depts) { + List deptTrees = buildDeptTree(depts); + List treeSelects = deptTrees.stream().map(TreeSelect::new).collect(Collectors.toList()); + return treeSelects; + } + + /** + * 构建前端所需要树结构 + * + * @param depts 部门列表 + * @return 树结构列表 + */ + @Override + public List buildDeptTree(List depts) { + List returnList = new ArrayList(); + List tempList = new ArrayList(); + for (DssAdminDept dept : depts) { + tempList.add(dept.getId()); + } + for (Iterator iterator = depts.iterator(); iterator.hasNext(); ) { + DssAdminDept dept = (DssAdminDept) iterator.next(); + // 如果是顶级节点, 遍历该父节点的所有子节点 + if (!tempList.contains(dept.getParentId())) { + recursionFn(depts, dept); + returnList.add(dept); + } + } + if (returnList.isEmpty()) { + returnList = depts; + } + return returnList; + + } + + @Override + public DssAdminDept selectDeptById(Long deptId) { + return dssAdminDeptMapper.selectDeptById(deptId); + } + + /** + * 是否存在子节点 + * + * @param deptId 部门ID + * @return 结果 + */ + @Override + public boolean hasChildById(Long deptId) { + int result = dssAdminDeptMapper.hasChildById(deptId); + return result > 0 ? true : false; + } + + /** + * 根据ID查询所有子部门(正常状态) + * + * @param deptId 部门ID + * @return 子部门数 + */ + @Override + public int selectNormalChildrenDeptById(Long deptId) { + return dssAdminDeptMapper.selectNormalChildrenDeptById(deptId); + } + + /** + * 查询部门是否存在用户 + * + * @param deptId 部门ID + * @return 结果 true 存在 false 不存在 + */ + @Override + public boolean checkDeptExistUser(Long deptId) { + int result = dssAdminDeptMapper.checkDeptExistUser(deptId); + return result > 0 ? true : false; + } + + + /** + * 修改保存部门信息 + * + * @param dept 部门信息 + * @return 结果 + */ + @Override + public int updateDept(DssAdminDept dept) { + DssAdminDept newParentDept = dssAdminDeptMapper.selectDeptById(dept.getParentId()); + DssAdminDept oldDept = dssAdminDeptMapper.selectDeptById(dept.getId()); + if (StringUtils.isNotNull(newParentDept) && StringUtils.isNotNull(oldDept)) { + String newAncestors = newParentDept.getAncestors() + "," + newParentDept.getId(); + String oldAncestors = oldDept.getAncestors(); + dept.setAncestors(newAncestors); + updateDeptChildren(dept.getId(), newAncestors, oldAncestors); + } + int result = dssAdminDeptMapper.updateDept(dept); + if (UserConstants.DEPT_NORMAL.equals(dept.getStatus())) { + // 如果该部门是启用状态,则启用该部门的所有上级部门 + updateParentDeptStatus(dept); + } + return result; + } + + @Override + public int deleteDeptById(Long id) { + return dssAdminDeptMapper.deleteDeptById(id); + } + + @Override + public boolean checkDeptFinalStage(Long parentId) { + + int result = dssAdminDeptMapper.checkDeptFinalStage(parentId); + return result > 2 ? true : false; + + } + + /** + * 修改子元素关系 + * + * @param deptId 被修改的部门ID + * @param newAncestors 新的父ID集合 + * @param oldAncestors 旧的父ID集合 + */ + public void updateDeptChildren(Long deptId, String newAncestors, String oldAncestors) { + List children = dssAdminDeptMapper.selectChildrenDeptById(deptId); + for (DssAdminDept child : children) { + child.setAncestors(child.getAncestors().replaceFirst(oldAncestors, newAncestors)); + } + if (children.size() > 0) { + dssAdminDeptMapper.updateDeptChildren(children); + } + } + + /** + * 修改该部门的父级部门状态 + * + * @param dept 当前部门 + */ + private void updateParentDeptStatus(DssAdminDept dept) { + String updateBy = dept.getUpdateBy(); + dept = dssAdminDeptMapper.selectDeptById(dept.getId()); + dept.setUpdateBy(updateBy); + dssAdminDeptMapper.updateDeptStatus(dept); + } + + /** + * 递归列表 + */ + private void recursionFn(List list, DssAdminDept t) { + // 得到子节点列表 + List childList = getChildList(list, t); + t.setChildren(childList); + for (DssAdminDept tChild : childList) { + if (hasChild(list, tChild)) { + recursionFn(list, tChild); + } + } + } + + /** + * 判断是否有子节点 + */ + private boolean hasChild(List list, DssAdminDept t) { + return getChildList(list, t).size() > 0 ? true : false; + } + + /** + * 得到子节点列表 + */ + private List getChildList(List list, DssAdminDept t) { + List tlist = new ArrayList(); + Iterator it = list.iterator(); + while (it.hasNext()) { + DssAdminDept n = (DssAdminDept) it.next(); + if (StringUtils.isNotNull(n.getParentId()) && n.getParentId().longValue() == t.getId().longValue()) { + tlist.add(n); + } + } + return tlist; + } + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssAdminUserServiceImpl.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssAdminUserServiceImpl.java new file mode 100644 index 000000000..5e338aa6e --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssAdminUserServiceImpl.java @@ -0,0 +1,212 @@ +package com.webank.wedatasphere.dss.framework.admin.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlySSOAppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.framework.admin.common.constant.UserConstants; +import com.webank.wedatasphere.dss.framework.admin.common.utils.StringUtils; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminUser; +import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; +import com.webank.wedatasphere.dss.framework.admin.xml.DssUserMapper; +import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkWarnException; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.user.SSOUserGetOperation; +import com.webank.wedatasphere.dss.standard.app.sso.user.SSOUserOperation; +import com.webank.wedatasphere.dss.standard.app.sso.user.SSOUserService; +import com.webank.wedatasphere.dss.standard.app.sso.user.ref.DSSUserContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.utils.RequestRefUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.function.Function; + +import static com.webank.wedatasphere.dss.framework.admin.conf.AdminConf.SUPER_ADMIN_LIST; + +@Service +public class DssAdminUserServiceImpl extends ServiceImpl implements DssAdminUserService { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Resource + DssUserMapper dssUserMapper; + + /** + * 校验用户名称是否唯一 + * + * @param userName 用户名称 + * @return 结果 + */ + @Override + public String checkUserNameUnique(String userName) { + int count = dssUserMapper.checkUserNameUnique(userName); + if (count > 0) { + return UserConstants.NOT_UNIQUE; + } + return UserConstants.UNIQUE; + } + + /** + * 校验用户名称是否唯一 + * + * @param user 用户信息 + * @return + */ + @Override + public String checkPhoneUnique(DssAdminUser user) { + Long userId = StringUtils.isNull(user.getId()) ? -1L : user.getId(); + DssAdminUser info = dssUserMapper.checkPhoneUnique(user.getPhonenumber()); + if (StringUtils.isNotNull(info) && info.getId().longValue() != userId.longValue()) { + return UserConstants.NOT_UNIQUE; + } + return UserConstants.UNIQUE; + } + + /** + * 校验email是否唯一 + * + * @param user 用户信息 + * @return + */ + @Override + public String checkEmailUnique(DssAdminUser user) { + Long userId = StringUtils.isNull(user.getId()) ? -1L : user.getId(); + DssAdminUser info = dssUserMapper.checkEmailUnique(user.getEmail()); + if (StringUtils.isNotNull(info) && info.getId().longValue() != userId.longValue()) { + return UserConstants.NOT_UNIQUE; + } + return UserConstants.UNIQUE; + } + + @Override + public void insertOrUpdateUser(String username, Workspace workspace) { + DssAdminUser dssUser = dssUserMapper.selectUserByName(username); + if (dssUser == null) { + logger.info("new user {} access to DSS, now try to add it.", username); + dssUser = new DssAdminUser(); + dssUser.setUserName(username); + dssUser.setName(username); + dssUser.setIsFirstLogin(1); + dssUser.setIsAdmin(ArrayUtils.contains(SUPER_ADMIN_LIST, username) ? 1 : 0); + dssUser.setLoginNum(1); + insertUser(dssUser, workspace); + } else { + dssUser.setIsFirstLogin(0); + dssUser.setLastLoginTime(new Date()); + dssUser.setLoginNum(dssUser.getLoginNum() == null ? 0 : dssUser.getLoginNum() + 1); + dssUserMapper.updateUser(dssUser); + } + } + + /** + * 新增保存用户信息 + * + * @param user 用户信息 + * @return 结果 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public int insertUser(DssAdminUser user, Workspace workspace) { + tryUserOperation((ssoUserService, requestRefUser) -> { + SSOUserGetOperation ssoUserOperation = ssoUserService.getSSOUserGetOperation(); + //一些没有实现该operation的如SSOAppconn + if (ssoUserOperation == null) { + return false; + } + DSSUserContentRequestRef requestRef = RequestRefUtils.getRequestRef(ssoUserOperation); + requestRef.setWorkspace(workspace).setUser(requestRefUser); + return StringUtils.isEmpty(ssoUserOperation.getUser(requestRef).getRefUserId()); + }, SSOUserService::getSSOUserCreationOperation, + (ssoUserCreationOperation, requestRef) -> ssoUserCreationOperation.createUser(requestRef), workspace, user); + return dssUserMapper.insertUser(user); + } + + @Override + public List selectUserList(DssAdminUser user) { + return dssUserMapper.selectUserList(user); + } + + /** + * 通过用户ID查询用户 + * + * @param userId 用户ID + * @return 用户对象信息 + */ + @Override + public DssAdminUser selectUserById(Long userId) { + return dssUserMapper.selectUserById(userId); + } + + @Override + public DssAdminUser selectUserByName(String username) { + return dssUserMapper.selectUserByName(username); + } + + + @Override + public int updateUser(DssAdminUser user, Workspace workspace) { + tryUserOperation(null, SSOUserService::getSSOUserUpdateOperation, + (ssoUserUpdateOperation, requestRef) -> ssoUserUpdateOperation.updateUser(requestRef), workspace, user); + return dssUserMapper.updateUser(user); + } + + private void tryUserOperation(BiPredicate filter, + Function operationFunction, + BiFunction operationConsumer, + Workspace workspace, DssAdminUser user) { + DSSUserContentRequestRef.User requestRefUser = new DSSUserContentRequestRef.User() { + @Override + public String getUsername() { + return user.getUserName(); + } + + @Override + public String getName() { + return user.getName(); + } + + @Override + public Boolean isFirstLogin() { + return user.getIsFirstLogin() == 1; + } + + @Override + public Boolean isAdmin() { + return user.getIsAdmin() == 1; + } + }; + AppConnManager.getAppConnManager().listAppConns().forEach(appConn -> { + if (appConn instanceof OnlySSOAppConn && CollectionUtils.isNotEmpty(appConn.getAppDesc().getAppInstances())) { + OnlySSOAppConn onlySSOAppConn = (OnlySSOAppConn) appConn; + appConn.getAppDesc().getAppInstances().forEach(appInstance -> { + SSOUserService ssoUserService = onlySSOAppConn.getOrCreateSSOStandard().getSSOUserService(appInstance); + if (filter != null && !filter.test(ssoUserService, requestRefUser)) { + return; + } + T ssoUserOperation = operationFunction.apply(ssoUserService); + if (ssoUserOperation == null) { + return; + } + DSSUserContentRequestRef requestRef = RequestRefUtils.getRequestRef(ssoUserOperation); + requestRef.setWorkspace(workspace).setUser(requestRefUser); + logger.info("try to ask AppConn {} to operate {} with user {}.", appConn.getAppDesc().getAppName(), ssoUserOperation.getClass().getSimpleName(), user); + ResponseRef responseRef = operationConsumer.apply(ssoUserOperation, requestRef); + if (responseRef.isFailed()) { + DSSExceptionUtils.dealWarnException(50030, responseRef.getErrorMsg(), DSSFrameworkWarnException.class); + } + }); + } + }); + } + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssProxyUserServiceImpl.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssProxyUserServiceImpl.java new file mode 100644 index 000000000..3e28e48b7 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssProxyUserServiceImpl.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.framework.admin.service.impl; + +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssProxyUser; +import com.webank.wedatasphere.dss.framework.admin.service.DssProxyUserService; +import com.webank.wedatasphere.dss.framework.admin.xml.DSSProxyUserMapper; +import org.springframework.stereotype.Service; +import javax.annotation.Resource; +import java.util.List; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.framework.admin.conf.AdminConf.DS_PROXY_SELF_ENABLE; + +@Service +public class DssProxyUserServiceImpl implements DssProxyUserService { + @Resource + DSSProxyUserMapper dssProxyUserMapper; + + @Override + public List selectProxyUserList(String userName) { + return dssProxyUserMapper.selectProxyUserList(userName); + } + + @Override + public List getProxyUserNameList(String userName) { + List userList = dssProxyUserMapper.selectProxyUserList(userName);; + List proxyUserNameList=userList.stream().map(dssProxyUser -> dssProxyUser.getProxyUserName()).collect(Collectors.toList()); + return proxyUserNameList; + } + + @Override + public int insertProxyUser(DssProxyUser dssProxyUser) { + int rows = dssProxyUserMapper.insertUser(dssProxyUser); + return rows; + } + + @Override + public boolean isExists(String userName, String proxyUserName) { + List res= dssProxyUserMapper.getProxyUserList(userName,proxyUserName); + if(DS_PROXY_SELF_ENABLE.getValue() && userName.equalsIgnoreCase(proxyUserName)){ + return true; + }else if(res.size()==0){ + return false; + }else { + return true; + } + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssScriptDownloadServiceImpl.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssScriptDownloadServiceImpl.java new file mode 100644 index 000000000..bddd3bd4b --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/DssScriptDownloadServiceImpl.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.framework.admin.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssScriptDownloadAudit; +import com.webank.wedatasphere.dss.framework.admin.service.DssScriptDownloadService; +import com.webank.wedatasphere.dss.framework.admin.xml.DssAuditMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @Auther: Han Tang + * @Date: 2022/1/11-01-11-15:29 + */ +@Service +public class DssScriptDownloadServiceImpl extends ServiceImpl implements DssScriptDownloadService { + @Autowired + public DssAuditMapper dssAuditMapper; + + public List getDownloadAuditList(String userName, String startIme, String endTime) { + return dssAuditMapper.getDownloadAuditList(userName, startIme, endTime); + } + + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/LdapServiceImpl.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/LdapServiceImpl.java new file mode 100644 index 000000000..395717494 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/service/impl/LdapServiceImpl.java @@ -0,0 +1,69 @@ +package com.webank.wedatasphere.dss.framework.admin.service.impl; + +import com.webank.wedatasphere.dss.framework.admin.service.LdapService; +import com.webank.wedatasphere.dss.framework.admin.util.LdapUtils; +import org.springframework.stereotype.Service; + +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.*; +import javax.naming.ldap.LdapContext; + +@Service +public class LdapServiceImpl implements LdapService { + + + /** add user to AD */ + @Override + public void addUser(String adminName,String adminPassword,String ldapUrl,String baseDN,String userName,String pwd) throws NamingException { + LdapContext ctx = LdapUtils.connectLDAP(adminName, adminPassword, ldapUrl); + BasicAttributes basicAttributes = new BasicAttributes(); + BasicAttribute objectClassSet = new BasicAttribute("objectclass"); + objectClassSet.add("inetOrgPerson"); + basicAttributes.put(objectClassSet); + basicAttributes.put("sn", userName); + basicAttributes.put("cn", userName); + basicAttributes.put("uid", userName); + basicAttributes.put("userPassword", pwd); + ctx.createSubcontext("uid="+userName+","+baseDN, basicAttributes); + LdapUtils.closeContext(ctx); + } + + + @Override + public void update(String adminName,String adminPassword,String ldapUrl,String baseDN,String userName,String pwd) throws NamingException { + LdapContext ctx = LdapUtils.connectLDAP(adminName, adminPassword, ldapUrl); + ModificationItem[] mods = new ModificationItem[1]; + Attribute attr = new BasicAttribute("userPassword", pwd); + // Support add, replace and remove an attribute. + mods[0] = new ModificationItem(DirContext.REPLACE_ATTRIBUTE, attr); + ctx.modifyAttributes("uid="+userName+","+baseDN, mods); + LdapUtils.closeContext(ctx); + + } + @Override + public boolean exist(String adminName,String adminPassword,String ldapUrl,String baseDN,String userName) throws NamingException { + LdapContext ctx = LdapUtils.connectLDAP(adminName, adminPassword, ldapUrl); + SearchControls searchCtls = new SearchControls(); + searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); + String searchFilter = "uid="+userName; + String searchBase = baseDN; + String returnedAtts[] = { "cn" }; + searchCtls.setReturningAttributes(returnedAtts); + boolean exist = false; + + NamingEnumeration entries = ctx.search(searchBase, searchFilter, searchCtls); + if(entries.hasMore()){ + exist = true; + } + LdapUtils.closeContext(ctx); + return exist; + + } + + + public static void main(String[] args) throws NamingException { + + } + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/util/LdapUtils.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/util/LdapUtils.java new file mode 100644 index 000000000..13a67403b --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/util/LdapUtils.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.framework.admin.util; + +import javax.naming.Context; +import javax.naming.NamingException; +import javax.naming.ldap.InitialLdapContext; +import javax.naming.ldap.LdapContext; +import java.util.Hashtable; + +public class LdapUtils { + public static LdapContext connectLDAP(String adminName, String adminPassword, String ldapUrl) throws NamingException { + + Hashtable HashEnv = new Hashtable(); + HashEnv.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); + HashEnv.put(Context.SECURITY_AUTHENTICATION, "simple");// "none","simple","strong" + HashEnv.put(Context.SECURITY_PRINCIPAL, adminName); + HashEnv.put(Context.SECURITY_CREDENTIALS, adminPassword); + HashEnv.put(Context.PROVIDER_URL, ldapUrl); + + LdapContext ctx = new InitialLdapContext(HashEnv, null); + return ctx; + } + + + public static void closeContext(LdapContext ctx) throws NamingException { + if (ctx != null) { + ctx.close(); + } + } +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/util/OkHttpHelper.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/util/OkHttpHelper.java new file mode 100644 index 000000000..6b887d320 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/util/OkHttpHelper.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.dss.framework.admin.util; + +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +/** + * @Auther: Han Tang + * @Date: 2022/1/19-01-19-15:35 + */ +public class OkHttpHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(OkHttpHelper.class); + public static int DEFAULT_TIMEOUT = 5; + private OkHttpClient okHttpClient; + + private OkHttpHelper() { + okHttpClient = new OkHttpClient(); + okHttpClient.setConnectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS); + } + + public static OkHttpHelper getInstance() { + return HttpHelperHolder.instance; + } + + public OkHttpClient getOkHttpClient() { + return okHttpClient; + } + + private static class HttpHelperHolder { + private static OkHttpHelper instance = new OkHttpHelper(); + } + + + public static Response syncGet(Request request) throws Exception { + OkHttpClient okHttpClient = OkHttpHelper.getInstance().getOkHttpClient(); + Response response = okHttpClient.newCall(request).execute(); + if (response.isSuccessful()) { + return response; + } else { + LOGGER.error("http error url:" + request.httpUrl().toString(), response); + throw new Exception(request.httpUrl().toString() + " http调用失败"); + } + } + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DSSProxyUserMapper.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DSSProxyUserMapper.java new file mode 100644 index 000000000..eadb667cd --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DSSProxyUserMapper.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.framework.admin.xml; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminUser; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssProxyUser; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +@Mapper +public interface DSSProxyUserMapper extends BaseMapper { + public List selectProxyUserList(String userName); + public int insertUser(DssProxyUser user); + public List getProxyUserList(@Param("userName") String userName, @Param("proxyUserName") String proxyUserName); + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssAdminDeptMapper.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssAdminDeptMapper.java new file mode 100644 index 000000000..cf074d18f --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssAdminDeptMapper.java @@ -0,0 +1,110 @@ +package com.webank.wedatasphere.dss.framework.admin.xml; + +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminDept; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +@Mapper +public interface DssAdminDeptMapper extends BaseMapper { + /** + * 查询部门管理数据 + * + * @param dept 部门信息 + * @return 部门信息集合 + */ + public List selectDeptList(DssAdminDept dept); + + + /** + * 根据部门ID查询信息 + * + * @param deptId 部门ID + * @return 部门信息 + */ + public DssAdminDept selectDeptById(Long deptId); + + /** + * 根据ID查询所有子部门 + * + * @param deptId 部门ID + * @return 部门列表 + */ + public List selectChildrenDeptById(Long deptId); + + /** + * 根据ID查询所有子部门(正常状态) + * + * @param deptId 部门ID + * @return 子部门数 + */ + public int selectNormalChildrenDeptById(Long deptId); + + /** + * 是否存在子节点 + * + * @param deptId 部门ID + * @return 结果 + */ + public int hasChildById(Long deptId); + + /** + * 查询部门是否存在用户 + * + * @param deptId 部门ID + * @return 结果 + */ + public int checkDeptExistUser(Long deptId); + + /** + * 校验部门名称是否唯一 + * + * @param deptName 部门名称 + * @param parentId 父部门ID + * @return 结果 + */ + public DssAdminDept checkDeptNameUnique(@Param("deptName") String deptName, @Param("parentId") Long parentId); + + /** + * 新增部门信息 + * + * @param dept 部门信息 + * @return 结果 + */ + public int insertDept(DssAdminDept dept); + + /** + * 修改部门信息 + * + * @param dept 部门信息 + * @return 结果 + */ + public int updateDept(DssAdminDept dept); + + /** + * 修改所在部门的父级部门状态 + * + * @param dept 部门 + */ + public void updateDeptStatus(DssAdminDept dept); + + /** + * 修改子元素关系 + * + * @param depts 子元素 + * @return 结果 + */ + public int updateDeptChildren(@Param("depts") List depts); + + /** + * 删除部门管理信息 + * + * @param deptId 部门ID + * @return 结果 + */ + public int deleteDeptById(Long deptId); + + public int checkDeptFinalStage(Long parentId); +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssAuditMapper.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssAuditMapper.java new file mode 100644 index 000000000..d0d49be84 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssAuditMapper.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.framework.admin.xml; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssScriptDownloadAudit; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +public interface DssAuditMapper extends BaseMapper { + + List getDownloadAuditList(@Param("creator") String creator,@Param("startTime") String startIme,@Param("endTime") String endTime); + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssUserMapper.java b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssUserMapper.java new file mode 100644 index 000000000..3c53b1d02 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/DssUserMapper.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.dss.framework.admin.xml; + +import com.webank.wedatasphere.dss.framework.admin.pojo.entity.DssAdminUser; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +@Mapper +public interface DssUserMapper extends BaseMapper { + + /** + * 根据条件分页查询用户列表 + * + * @param DssAdminUser 用户信息 + * @return 用户信息集合信息 + */ + List selectUserList(DssAdminUser DssAdminUser); + + + /** + * 通过用户ID查询用户 + * + * @param id 用户ID + * @return 用户对象信息 + */ + DssAdminUser selectUserById(Long id); + + DssAdminUser selectUserByName(String username); + + /** + * 新增用户信息 + * + * @param user 用户信息 + * @return 结果 + */ + int insertUser(DssAdminUser user); + + /** + * 修改用户信息 + * + * @param user 用户信息 + * @return 结果 + */ + int updateUser(DssAdminUser user); + + + + /** + * 校验用户名称是否唯一 + * + * @param userName 用户名称 + * @return 结果 + */ + int checkUserNameUnique(String userName); + + /** + * 校验手机号码是否唯一 + * + * @param phonenumber 手机号码 + * @return 结果 + */ + DssAdminUser checkPhoneUnique(String phonenumber); + + /** + * 校验email是否唯一 + * + * @param email 用户邮箱 + * @return 结果 + */ + DssAdminUser checkEmailUnique(String email); + + + + +} diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssAdminDeptMapper.xml b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssAdminDeptMapper.xml new file mode 100644 index 000000000..7e609ec75 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssAdminDeptMapper.xml @@ -0,0 +1,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + select d.id, d.parent_id, d.ancestors, d.dept_name, d.order_num, d.leader, d.phone, d.email, d.status, d.del_flag, d.create_by, d.create_time + from dss_workspace_admin_dept d + + + + + + + + + + + + + + + + + + + + insert into dss_workspace_admin_dept( + id, + parent_id, + dept_name, + ancestors, + order_num, + leader, + phone, + email, + status, + create_by, + create_time + )values( + #{id}, + #{parentId}, + #{deptName}, + #{ancestors}, + #{orderNum}, + #{leader}, + #{phone}, + #{email}, + #{status}, + #{createBy}, + sysdate() + ) + + + + update dss_workspace_admin_dept + + parent_id = #{parentId}, + dept_name = #{deptName}, + ancestors = #{ancestors}, + order_num = #{orderNum}, + leader = #{leader}, + phone = #{phone}, + email = #{email}, + status = #{status}, + update_by = #{updateBy}, + update_time = sysdate() + + where id = #{id} + + + + update dss_workspace_admin_dept set ancestors = + + when #{item.id} then #{item.ancestors} + + where id in + + #{item.id} + + + + + update dss_workspace_admin_dept + + status = #{status}, + update_by = #{updateBy}, + update_time = sysdate() + + where id in (${ancestors}) + + + + update dss_workspace_admin_dept set del_flag = '2' where id = #{id} + + + + + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssAuditMapper.xml b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssAuditMapper.xml new file mode 100644 index 000000000..3030bc3f2 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssAuditMapper.xml @@ -0,0 +1,31 @@ + + + + + + + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssProxyUserMapper.xml b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssProxyUserMapper.xml new file mode 100644 index 000000000..aafdd917c --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssProxyUserMapper.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + insert into dss_proxy_user( + id, + username, + proxy_user_name, + create_by, + remark, + create_time + )values( + #{id}, + #{userName}, + #{proxyUserName}, + #{createBy}, + #{remark}, + sysdate() + ) + + + + diff --git a/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssUserMapper.xml b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssUserMapper.xml new file mode 100644 index 000000000..65c934193 --- /dev/null +++ b/dss-framework/dss-framework-admin-service/src/main/java/com/webank/wedatasphere/dss/framework/admin/xml/impl/DssUserMapper.xml @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + select u.id, u.username, u.name, u.is_first_login, u.dept_id, u.is_admin, u.email, u.phonenumber, u.password, u.del_flag, u.create_time, u.update_time, u.remark, + d.id , d.parent_id, d.dept_name, d.order_num, d.leader, d.status as dept_status + from dss_user u + left join dss_workspace_admin_dept d on u.dept_id = d.id + + + + + + + + + + + + + + update dss_user + + name = #{name}, + dept_id = #{deptId}, + email = #{email}, + phonenumber = #{phonenumber}, + password = #{password}, + + + where id = #{id} + + + + + + + + + + + insert into dss_user( + id, + dept_id, + username, + name, + email, + phonenumber, + password, + create_by, + create_time + )values( + #{id}, + #{deptId}, + #{userName}, + #{name}, + #{email}, + #{phonenumber}, + #{password}, + #{createBy}, + sysdate() + ) + + + + diff --git a/dss-framework/dss-framework-common/pom.xml b/dss-framework/dss-framework-common/pom.xml new file mode 100644 index 000000000..17bacb460 --- /dev/null +++ b/dss-framework/dss-framework-common/pom.xml @@ -0,0 +1,39 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + + 4.0.0 + dss-framework-common + jar + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkErrorException.java b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkErrorException.java new file mode 100644 index 000000000..e44bcf0f7 --- /dev/null +++ b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkErrorException.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.common.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class DSSFrameworkErrorException extends ErrorException { + + public DSSFrameworkErrorException(int errorCode, String errorDesc){ + super(errorCode, errorDesc); + } + + public static void dealErrorException(int errorCode, String errorDesc, Throwable t) throws DSSFrameworkErrorException{ + DSSFrameworkErrorException dssFrameworkErrorException = new DSSFrameworkErrorException(errorCode, errorDesc); + dssFrameworkErrorException.initCause(t); + throw dssFrameworkErrorException; + } + + public static void dealErrorException(int errorCode, String errorDesc) throws DSSFrameworkErrorException{ + throw new DSSFrameworkErrorException(errorCode, errorDesc); + } + + +} diff --git a/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkRuntimeException.java b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkRuntimeException.java new file mode 100644 index 000000000..4675dd9dc --- /dev/null +++ b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkRuntimeException.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.common.exception; + +import org.apache.linkis.common.exception.WarnException; + + +public class DSSFrameworkRuntimeException extends WarnException { + + public DSSFrameworkRuntimeException(String msg){ + super(100000, msg); + } +} diff --git a/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkWarnException.java b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkWarnException.java new file mode 100644 index 000000000..04b10f37b --- /dev/null +++ b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/DSSFrameworkWarnException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.common.exception; + +import org.apache.linkis.common.exception.WarnException; + + +public class DSSFrameworkWarnException extends WarnException { + public DSSFrameworkWarnException(int errorCode, String errorDesc){ + super(errorCode, errorDesc); + } + + public static void dealWarnException(int errorCode, String errorDesc, Throwable t) throws DSSFrameworkWarnException { + DSSFrameworkWarnException dssFrameworkWarnException = new DSSFrameworkWarnException(errorCode, errorDesc); + dssFrameworkWarnException.initCause(t); + throw dssFrameworkWarnException; + } + +} diff --git a/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/ThrowingConsumer.java b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/ThrowingConsumer.java new file mode 100644 index 000000000..56cf17087 --- /dev/null +++ b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/ThrowingConsumer.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.common.exception; + + +@FunctionalInterface +public interface ThrowingConsumer { + void accept(T t) throws E; +} diff --git a/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/ThrowingFunction.java b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/ThrowingFunction.java new file mode 100644 index 000000000..54e1f3b37 --- /dev/null +++ b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/exception/ThrowingFunction.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.common.exception; + + +@FunctionalInterface +public interface ThrowingFunction { + R accept(T t) throws E; +} diff --git a/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/utils/DSSFrameworkExceptionUtils.java b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/utils/DSSFrameworkExceptionUtils.java new file mode 100644 index 000000000..24b846893 --- /dev/null +++ b/dss-framework/dss-framework-common/src/main/java/com/webank/wedatasphere/dss/framework/common/utils/DSSFrameworkExceptionUtils.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.common.utils; + + +import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkRuntimeException; +import com.webank.wedatasphere.dss.framework.common.exception.ThrowingConsumer; +import com.webank.wedatasphere.dss.framework.common.exception.ThrowingFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.Consumer; +import java.util.function.Function; + + +public class DSSFrameworkExceptionUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSFrameworkExceptionUtils.class); + + public static Consumer handling( + ThrowingConsumer throwingConsumer) { + return i -> { + try { + throwingConsumer.accept(i); + } catch (Exception e) { + LOGGER.error("execute failed,reason:",e); + throw new DSSFrameworkRuntimeException(e.getMessage()); + } + }; + } + + public static Function map( + ThrowingFunction throwingFunction) { + return i -> { + try { + return throwingFunction.accept(i); + } catch (Exception e) { + LOGGER.error("execute failed,reason:",e); + throw new DSSFrameworkRuntimeException(e.getMessage()); + } + }; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/pom.xml b/dss-framework/dss-framework-orchestrator-server/pom.xml new file mode 100644 index 000000000..3f3ef47b2 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/pom.xml @@ -0,0 +1,236 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + + 4.0.0 + + dss-framework-orchestrator-server + + + + com.webank.wedatasphere.dss + dss-framework-common + ${dss.version} + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + + + commons-beanutils + commons-beanutils + + + commons-collections + commons-collections + + + + + com.webank.wedatasphere.dss + dss-orchestrator-db + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-orchestrator-loader + ${dss.version} + + + com.webank.wedatasphere.dss + dss-framework-orchestrator-publish + ${dss.version} + + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + org.springframework.boot + spring-boot-starter-log4j2 + + + + + + com.webank.wedatasphere.dss + dss-contextservice + ${dss.version} + + + linkis-common + org.apache.linkis + + + commons-text + org.apache.commons + + + json4s-jackson_2.11 + org.json4s + + + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + com.webank.wedatasphere.dss + dss-appconn-manager-client + ${dss.version} + + + org.apache.commons + commons-math3 + provided + + + com.webank.wedatasphere.dss + dss-sender-service + ${dss.version} + provided + + + org.apache.linkis + linkis-storage-script-dev-server + ${linkis.version} + + + spring-context-support + org.springframework + + + + + org.apache.linkis + linkis-storage + ${linkis.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + **/*.properties + **/*.yml + + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/assembly/distribution.xml b/dss-framework/dss-framework-orchestrator-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..313ea1281 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/assembly/distribution.xml @@ -0,0 +1,44 @@ + + + + dss-framework-orchestrator-server + + dir + + true + dss-framework-orchestrator-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java new file mode 100644 index 000000000..7c830dd65 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.conf; + +import org.apache.linkis.common.conf.CommonVars; + + +public class OrchestratorConf { + public static final CommonVars DSS_UPLOAD_PATH = CommonVars.apply("wds.dss.file.upload.dir", "/appcom/tmp/uploads"); +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java new file mode 100644 index 000000000..e2df01daa --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.conf; + +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class OrchestratorSpringConf { + + @Bean(name = "contextService") + public ContextService createContextService(){ + return ContextServiceImpl.getInstance(); + } + +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java new file mode 100644 index 000000000..e4a1dd646 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.constant; + + +import com.webank.wedatasphere.dss.orchestrator.publish.job.OrchestratorConversionJob; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class DSSOrchestratorConstant { + public static final String PUBLISH_FLOW_REPORT_FORMATE = "工作流名:%s,版本号:%s,工作流内容为空,请自行修改或者删除"; + + public static Map orchestratorConversionJobMap = new ConcurrentHashMap<>(); +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorLevelEnum.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorLevelEnum.java new file mode 100644 index 000000000..6e9a3bf8f --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorLevelEnum.java @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.orchestrator.server.constant; + +public enum OrchestratorLevelEnum { + HIGH, MEDIUM, LOW +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorSpringConfiguration.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorSpringConfiguration.java new file mode 100644 index 000000000..b144a7a79 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorSpringConfiguration.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.constant; + +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorFrameworkService; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorPluginService; +import com.webank.wedatasphere.dss.orchestrator.server.service.impl.OrchestratorFrameworkServiceImpl; +import com.webank.wedatasphere.dss.orchestrator.server.service.impl.OrchestratorPluginServiceImpl; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class OrchestratorSpringConfiguration { + + @Bean + @ConditionalOnMissingBean + public OrchestratorPluginService createOrchestratorPluginService() { + return new OrchestratorPluginServiceImpl(); + } + + @Bean + @ConditionalOnMissingBean + public OrchestratorFrameworkService createOrchestratorFrameworkService() { + return new OrchestratorFrameworkServiceImpl(); + } + +} \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/UploadFileResponse.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/UploadFileResponse.java new file mode 100644 index 000000000..d1d72a6c1 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/UploadFileResponse.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.entity; + +public class UploadFileResponse { + private String fileName; + private String fileDownloadUri; + private String fileType; + private long size; + + public UploadFileResponse(String fileName, String fileDownloadUri, String fileType, long size) { + this.fileName = fileName; + this.fileDownloadUri = fileDownloadUri; + this.fileType = fileType; + this.size = size; + } + // getter and setter ... +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/AddOrchestratorRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/AddOrchestratorRequest.java new file mode 100644 index 000000000..1ca831b51 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/AddOrchestratorRequest.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +@Deprecated +public class AddOrchestratorRequest { + private String name; + private String workspaceName; + private String projectName; + private String type; + private String desc; + private Long projectID; + + private LabelRouteVO labels; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public Long getProjectID() { + return projectID; + } + + public void setProjectID(Long projectID) { + this.projectID = projectID; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OpenOrchestratorRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OpenOrchestratorRequest.java new file mode 100644 index 000000000..aee28c2f8 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OpenOrchestratorRequest.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class OpenOrchestratorRequest { + + private String workspaceName; + private Long orchestratorId; + + private LabelRouteVO labels; + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java new file mode 100644 index 000000000..b543b0a8a --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java @@ -0,0 +1,148 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + + +@XmlRootElement +public class OrchestratorCreateRequest extends OrchestratorRequest { + + @NotNull(message = "编排名称不能为空") + private String orchestratorName; + + /** + * 编排模式,如工作流,组合编排等 + */ + @NotNull(message = "编排模式类型不能为空") + private String orchestratorMode; + + /** + * 编排方式 + */ + @NotNull(message = "编排方式不能为空") + private List orchestratorWays; + + /** + * 编排重要性级别 + */ + private String orchestratorLevel; + + private List dssLabels; + + /** + * 编排用途 + */ + private String uses; + + @NotNull(message = "描述不能为空") + private String description; + + private String projectName; + + private String workspaceName; + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + @Override + public String getOrchestratorMode() { + return orchestratorMode; + } + + @Override + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } + + public List getOrchestratorWays() { + return orchestratorWays; + } + + public void setOrchestratorWays(List orchestratorWays) { + this.orchestratorWays = orchestratorWays; + } + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getOrchestratorLevel() { + return orchestratorLevel; + } + + public void setOrchestratorLevel(String orchestratorLevel) { + this.orchestratorLevel = orchestratorLevel; + } + + @Override + public String toString() { + return "OrchestratorCreateRequest{" + + "workspaceId=" + getWorkspaceId() + + ", projectId=" + getProjectId() + + ", arrangeName='" + orchestratorName + '\'' + + ", arrangeMode='" + orchestratorMode + '\'' + + ", arrangeWays=" + orchestratorWays + + ", uses='" + uses + '\'' + + ", description='" + description + '\'' + + ", labels=" + getLabels() + + '}'; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorDeleteRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorDeleteRequest.java new file mode 100644 index 000000000..1302926e3 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorDeleteRequest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; + + +@XmlRootElement +public class OrchestratorDeleteRequest extends OrchestratorRequest { + + @NotNull(message = "id不能为空") + private Long id; + + private Boolean deleteSchedulerWorkflow = false; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Boolean getDeleteSchedulerWorkflow() { + return deleteSchedulerWorkflow; + } + + public void setDeleteSchedulerWorkflow(Boolean deleteSchedulerWorkflow) { + this.deleteSchedulerWorkflow = deleteSchedulerWorkflow; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java new file mode 100644 index 000000000..b454d1d5b --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java @@ -0,0 +1,134 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + +@XmlRootElement +public class OrchestratorModifyRequest extends OrchestratorRequest { + + @NotNull(message = "id不能为空") + private Long id; + + @NotNull(message = "编排名称不能为空") + private String orchestratorName; + + /** + * 编排模式,如工作流,组合编排等 + */ + @NotNull(message = "编排模式不能为空") + private String orchestratorMode; + + /** + * 编排方式 + */ + @NotNull(message = "编排方式不能为空") + private List orchestratorWays; + + private String orchestratorLevel; + + /** + * 编排用途 + */ + private String uses; + + @NotNull(message = "描述不能为空") + private String description; + + private List dssLabels; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public String getOrchestratorMode() { + return orchestratorMode; + } + + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } + + public List getOrchestratorWays() { + return orchestratorWays; + } + + public void setOrchestratorWays(List orchestratorWays) { + this.orchestratorWays = orchestratorWays; + } + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + @Override + public String toString() { + return "OrchestratorModifyRequest{" + + "id=" + id + + ", workspaceId=" + getWorkspaceId() + + ", projectId=" + getProjectId() + + ", orchestratorName='" + orchestratorName + '\'' + + ", orchestratorMode='" + orchestratorMode + '\'' + + ", orchestratorWays=" + orchestratorWays + + ", uses='" + uses + '\'' + + ", description='" + description + '\'' + + ", labels=" + getLabels() + + '}'; + } + + public String getOrchestratorLevel() { + return orchestratorLevel; + } + + public void setOrchestratorLevel(String orchestratorLevel) { + this.orchestratorLevel = orchestratorLevel; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java new file mode 100644 index 000000000..3c22462cb --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; + + +@XmlRootElement +public class OrchestratorRequest { + + @NotNull(message = "workspaceId不能为空") + private Long workspaceId; + + @NotNull(message = "工程id不能为空") + private Long projectId; + + /** + * 编排类型,如工作流,组合编排等 + */ + // @NotNull(message = "编排类型不能为空") + private String orchestratorMode; + + /** + * dssLabels是通过前端进行传入的,主要是用来进行当前的环境信息 + */ + private LabelRouteVO labels; + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getOrchestratorMode() { + return orchestratorMode; + } + + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } + + @Override + public String toString() { + return "OrchestratorRequest{" + + ", workspaceId=" + workspaceId + + ", projectId=" + projectId + + ", orchestratorMode='" + orchestratorMode + '\'' + + '}'; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/QueryOrchestratorVersion.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/QueryOrchestratorVersion.java new file mode 100644 index 000000000..4debd5ec0 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/QueryOrchestratorVersion.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class QueryOrchestratorVersion { + + @NotNull(message = "orchestratorId不能为空") + private Long orchestratorId; + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/RollbackOrchestratorRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/RollbackOrchestratorRequest.java new file mode 100644 index 000000000..e26bb0e12 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/RollbackOrchestratorRequest.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class RollbackOrchestratorRequest { + private Long orchestratorId; + private String version; + private Long projectId; + private String projectName; + + private LabelRouteVO labels; + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/CommonOrchestratorVo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/CommonOrchestratorVo.java new file mode 100644 index 000000000..f07198585 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/CommonOrchestratorVo.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; + + +public class CommonOrchestratorVo { + + final static String WORK_FLOW = "workflow"; + + final static String ORDINARY = "ordinary"; + + private Long orchestratorId; + + private String orchestratorVersion; + + private String type = WORK_FLOW; + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getOrchestratorVersion() { + return orchestratorVersion; + } + + public void setOrchestratorVersion(String orchestratorVersion) { + this.orchestratorVersion = orchestratorVersion; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java new file mode 100644 index 000000000..953307bec --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java @@ -0,0 +1,254 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; + +import java.util.Date; +import java.util.List; + +public class OrchestratorBaseInfo { + /** + * 主键ID + */ + private Long id; + + /** + * 空间id + */ + private Long workspaceId; + + /** + * 工程id + */ + private Long projectId; + + /** + * 编排模式id(工作流,调用orchestrator服务返回的orchestratorId) + */ + private Long orchestratorId; + + /** + * 编排模式版本id(工作流,调用orchestrator服务返回的orchestratorVersionId) + */ + private Long orchestratorVersionId; + + /** + * 编排名称 + */ + private String orchestratorName; + + /** + * 编排模式,取得的值是dss_dictionary中的dic_key(parent_key=p_orchestratorment_mode) + */ + private String orchestratorMode; + + /** + * 编排方式 + */ + //private String orchestratorWay; + + /** + * 用途 + */ + private String uses; + + /** + * 描述 + */ + private String description; + + /** + * 创建人 + */ + private String createUser; + + /** + * 创建时间 + */ + private Date createTime; + + /** + * 更新人 + */ + private String updateUser; + + /** + * 更新时间 + */ + private Date updateTime; + + private List orchestratorWays; + + private String orchestratorLevel; + + private boolean flowEditLockExist = false; + + public Boolean getEditable() { + return editable; + } + + public void setEditable(Boolean editable) { + this.editable = editable; + } + + public Boolean getReleasable() { + return releasable; + } + + public void setReleasable(Boolean releasable) { + this.releasable = releasable; + } + + /** + * 工程权限等级:0-查看,1-编辑,2-发布 + */ + + private Boolean editable; + + /** + * 工作流是否可发布 + */ + private Boolean releasable; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Long getOrchestratorVersionId() { + return orchestratorVersionId; + } + + public void setOrchestratorVersionId(Long orchestratorVersionId) { + this.orchestratorVersionId = orchestratorVersionId; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public String getOrchestratorMode() { + return orchestratorMode; + } + + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public List getOrchestratorWays() { + return orchestratorWays; + } + + public void setOrchestratorWays(List orchestratorWays) { + this.orchestratorWays = orchestratorWays; + } + + public boolean isFlowEditLockExist() { + return flowEditLockExist; + } + + public void setFlowEditLockExist(boolean flowEditLockExist) { + this.flowEditLockExist = flowEditLockExist; + } + + public String getOrchestratorLevel() { + return orchestratorLevel; + } + + public void setOrchestratorLevel(String orchestratorLevel) { + this.orchestratorLevel = orchestratorLevel; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java new file mode 100644 index 000000000..fbbaa17eb --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java @@ -0,0 +1,137 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.restful; + +import com.webank.wedatasphere.dss.appconn.manager.utils.AppConnManagerUtils; +import com.webank.wedatasphere.dss.orchestrator.server.constant.OrchestratorLevelEnum; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorCreateRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorDeleteRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.CommonOrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorFrameworkService; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.PostConstruct; +import javax.servlet.http.HttpServletRequest; +import java.util.Arrays; +import java.util.List; + +@RequestMapping(path = "/dss/framework/orchestrator", produces = {"application/json"}) +@RestController +public class DSSFrameworkOrchestratorRestful { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSFrameworkOrchestratorRestful.class); + + @Autowired + private OrchestratorFrameworkService orchestratorFrameworkService; + @Autowired + private OrchestratorService orchestratorService; + + @PostConstruct + public void init() { + AppConnManagerUtils.autoLoadAppConnManager(); + } + + /** + * 创建编排模式 + * + * @param httpServletRequest + * @param createRequest + * @return + */ + @RequestMapping(path = "createOrchestrator", method = RequestMethod.POST) + public Message createOrchestrator(HttpServletRequest httpServletRequest, @RequestBody OrchestratorCreateRequest createRequest) throws Exception{ + String username = SecurityFilter.getLoginUsername(httpServletRequest); + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + LOGGER.info("workspace is {}", workspace.getWorkspaceName()); + //保存编排模式 + //todo 先注释掉 + // orchestratorService.saveOrchestrator(createRequest,null,username); + CommonOrchestratorVo orchestratorVo = orchestratorFrameworkService.createOrchestrator(username, createRequest, workspace); + return Message.ok("创建工作流编排模式成功").data("orchestratorId", orchestratorVo.getOrchestratorId()); + } + + /** + * 查询所有的编排模式 + * + * @param httpServletRequest + * @param orchestratorRequest + * @return + */ + @RequestMapping(path = "getAllOrchestrator", method = RequestMethod.POST) + public Message getAllOrchestrator(HttpServletRequest httpServletRequest, @RequestBody OrchestratorRequest orchestratorRequest) { + try { + String username = SecurityFilter.getLoginUsername(httpServletRequest); + return Message.ok("获取编排模式成功").data("page", orchestratorService.getListByPage(orchestratorRequest, username)); + } catch (Exception e) { + LOGGER.error("getAllOrchestratorError ", e); + return Message.error("获取编排模式失败:" + e.getMessage()); + } + } + + /** + * 修改编排模式 + * + * @param httpServletRequest + * @param modifyRequest + * @return + */ + @RequestMapping(path = "modifyOrchestrator", method = RequestMethod.POST) + public Message modifyOrchestrator(HttpServletRequest httpServletRequest, @RequestBody OrchestratorModifyRequest modifyRequest) throws Exception{ + String username = SecurityFilter.getLoginUsername(httpServletRequest); + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + LOGGER.info("workspace is {}", workspace.getWorkspaceName()); + CommonOrchestratorVo orchestratorVo = orchestratorFrameworkService.modifyOrchestrator(username, modifyRequest, workspace); + return Message.ok("修改工作流编排模式成功").data("orchestratorId", orchestratorVo.getOrchestratorId()); + } + + /** + * 删除编排模式 + * + * @param httpServletRequest + * @param deleteRequest + * @return + */ + @RequestMapping(path = "deleteOrchestrator", method = RequestMethod.POST) + public Message deleteOrchestrator(HttpServletRequest httpServletRequest, @RequestBody OrchestratorDeleteRequest deleteRequest) throws Exception { + String username = SecurityFilter.getLoginUsername(httpServletRequest); + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + orchestratorFrameworkService.deleteOrchestrator(username, deleteRequest, workspace); + return Message.ok("删除工作流编排模式成功"); + } + + @RequestMapping(path = "orchestratorLevels", method = RequestMethod.GET) + public Message getOrchestratorLevels(HttpServletRequest httpServletRequest) { + String username = SecurityFilter.getLoginUsername(httpServletRequest); + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + List levels = Arrays.asList(OrchestratorLevelEnum.values()); + return Message.ok("获取编排重要级别列表成功").data("orchestratorLevels", levels); + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java new file mode 100644 index 000000000..31293e1f4 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java @@ -0,0 +1,178 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.restful; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelKeyConvertor; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestImportOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext; +import com.webank.wedatasphere.dss.orchestrator.core.service.BMLService; +import com.webank.wedatasphere.dss.orchestrator.publish.ExportDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.publish.ImportDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.io.IOUtils; +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.filesystem.service.FsService; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.apache.linkis.storage.fs.FileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +@RequestMapping(path = "/dss/framework/orchestrator", produces = {"application/json"}) +@RestController +public class OrchestratorIERestful { + private static final Logger logger = LoggerFactory.getLogger(OrchestratorIERestful.class); + @Autowired + private BMLService bmlService; + @Autowired + private FsService fsService; + @Autowired + OrchestratorService orchestratorService; + @Autowired + private DSSOrchestratorContext orchestratorContext; + + @RequestMapping(path ="importOrchestratorFile", method = RequestMethod.POST) + public Message importOrcFile(HttpServletRequest req, + @RequestParam(required = false, name = "projectName") String projectName, + @RequestParam(required = false, name = "projectID") Long projectID, + @RequestParam(required = false, name = "labels") String labels, + @RequestParam(required = false, name = "packageFile") MultipartFile packageFile, + @RequestParam(required = false, name = "packageUri") String packageUri) throws Exception { + + if (null == packageFile && packageUri == null) { + throw new DSSErrorException(100788, "Import orchestrator failed for files is empty"); + } + String userName = SecurityFilter.getLoginUsername(req); + //调用工具类生产label + List dssLabelList = getDSSLabelList(labels); + Workspace workspace = SSOHelper.getWorkspace(req); + InputStream inputStream; + String fileName; + if (packageFile != null) { + inputStream = packageFile.getInputStream(); + fileName = new String(packageFile.getOriginalFilename().getBytes("ISO8859-1"), "UTF-8"); + //3、打包新的zip包上传BML + logger.info("User {} begin to import orchestrator file", userName); + } else { + FsPath fsPath = new FsPath(packageUri); + FileSystem fileSystem = fsService.getFileSystem(userName, fsPath); + if ( !fileSystem.exists(fsPath) ) { + throw new DSSRuntimeException("路径上不存在文件!"); + } + inputStream = fileSystem.read(fsPath); + fileName = packageUri.substring(packageUri.lastIndexOf('/') + 1); + } + + Map resultMap = bmlService.upload(userName, inputStream, fileName, projectName); + Long importOrcId; + try { + RequestImportOrchestrator importRequest = new RequestImportOrchestrator(userName, projectName, + projectID, resultMap.get("resourceId").toString(), + resultMap.get("version").toString(), null, dssLabelList, workspace); + importOrcId = orchestratorContext.getDSSOrchestratorPlugin(ImportDSSOrchestratorPlugin.class).importOrchestrator(importRequest); + } catch (Exception e) { + logger.error("Import orchestrator failed for ", e); + throw new DSSErrorException(100789, "Import orchestrator failed for " + e.getMessage()); + } + return Message.ok().data("importOrcId", importOrcId); + } + + @RequestMapping(path ="exportOrchestrator", method = RequestMethod.GET) + public void exportOrcFile(HttpServletRequest req, + HttpServletResponse resp, + @RequestParam(defaultValue = "exportOrc",required = false, name = "outputFileName") String outputFileName, + @RequestParam(defaultValue = "utf-8",required = false, name = "charset") String charset, + @RequestParam(defaultValue = "zip",required = false, name = "outputFileType") String outputFileType, + @RequestParam(name = "projectName") String projectName, + @RequestParam(name = "orchestratorId") Long orchestratorId, + @RequestParam(required = false, name = "orcVersionId") Long orcVersionId, + @RequestParam(defaultValue = "false",required = false, name = "addOrcVersion") Boolean addOrcVersion, + @RequestParam(required = false, name = "labels") String labels) throws DSSErrorException, IOException { + resp.addHeader("Content-Disposition", "attachment;filename=" + + new String(outputFileName.getBytes("UTF-8"), "ISO8859-1") + "." + outputFileType); + resp.setCharacterEncoding(charset); + Workspace workspace = SSOHelper.getWorkspace(req); + String userName = SecurityFilter.getLoginUsername(req); + List dssLabelList = getDSSLabelList(labels); + Map res = null; + OrchestratorVo orchestratorVo; + if (orcVersionId != null) { + orchestratorVo = orchestratorService.getOrchestratorVoByIdAndOrcVersionId(orchestratorId, orcVersionId); + } else { + orchestratorVo = orchestratorService.getOrchestratorVoById(orchestratorId); + } + orcVersionId = orchestratorVo.getDssOrchestratorVersion().getId(); + logger.info("export orchestrator orchestratorId " + orchestratorId + ",orcVersionId:" + orcVersionId); + try { + res = orchestratorContext.getDSSOrchestratorPlugin(ExportDSSOrchestratorPlugin.class).exportOrchestrator(userName, + orchestratorId, orcVersionId, projectName, dssLabelList, addOrcVersion, workspace); + } catch (Exception e) { + logger.error("export orchestrator failed for ", e); + throw new DSSErrorException(100789, "export orchestrator failed for " + e.getMessage()); + } + if (null != res) { + Map downRes = bmlService.download(userName, + res.get("resourceId").toString(), + res.get("version").toString()); + + InputStream inputStream = (InputStream) downRes.get("is"); + try { + IOUtils.copy(inputStream, resp.getOutputStream()); + resp.getOutputStream().flush(); + } finally { + IOUtils.closeQuietly(inputStream); + } + } + } + + //生成label list + public List getDSSLabelList(String labels) { + String labelStr = DSSCommonUtils.ENV_LABEL_VALUE_DEV; + try{ + Map labelMap = DSSCommonUtils.COMMON_GSON.fromJson(labels, Map.class); + if (labelMap.containsKey(LabelKeyConvertor.ROUTE_LABEL_KEY)) { + labelStr = (String) labelMap.get(LabelKeyConvertor.ROUTE_LABEL_KEY); + } + }catch (Exception e){ + logger.error("get labels failed for {}", e.getMessage()); + } + List dssLabelList = Arrays.asList(new EnvDSSLabel(labelStr)); + return dssLabelList; + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorRestful.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorRestful.java new file mode 100644 index 000000000..304aec383 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorRestful.java @@ -0,0 +1,96 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.restful; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OpenOrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.QueryOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.RollbackOrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import java.util.Arrays; +import java.util.List; + + +@RequestMapping(path = "/dss/framework/orchestrator", produces = {"application/json"}) +@RestController +public class OrchestratorRestful { + private final static Logger LOGGER = LoggerFactory.getLogger(OrchestratorRestful.class); + @Autowired + OrchestratorService orchestratorService; + + @RequestMapping(path = "rollbackOrchestrator", method = RequestMethod.POST) + public Message rollbackOrchestrator(HttpServletRequest request, @RequestBody RollbackOrchestratorRequest rollbackOrchestratorRequest) { + String username = SecurityFilter.getLoginUsername(request); + Long orchestratorId = rollbackOrchestratorRequest.getOrchestratorId(); + String version = rollbackOrchestratorRequest.getVersion(); + Long projectId = rollbackOrchestratorRequest.getProjectId(); + String projectName = rollbackOrchestratorRequest.getProjectName(); + Workspace workspace = SSOHelper.getWorkspace(request); + DSSLabel envDSSLabel = new EnvDSSLabel(rollbackOrchestratorRequest.getLabels().getRoute()); + try { + String newVersion = orchestratorService.rollbackOrchestrator(username, projectId, projectName, orchestratorId, version, envDSSLabel, workspace); + Message message = Message.ok("回滚版本成功").data("newVersion", newVersion); + return message; + } catch (final Throwable t) { + LOGGER.error("Failed to rollback orchestrator for user {} orchestratorId {}, projectId {} version {}", + username, orchestratorId, projectId, version, t); + return Message.error("回滚工作流版本失败"); + } + } + + @RequestMapping(path = "openOrchestrator", method = RequestMethod.POST) + public Message openOrchestrator(HttpServletRequest req, @RequestBody OpenOrchestratorRequest openOrchestratorRequest) throws Exception { + String openUrl = ""; + String userName = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + List dssLabelList = Arrays.asList(new EnvDSSLabel(openOrchestratorRequest.getLabels().getRoute())); + Long orchestratorId = openOrchestratorRequest.getOrchestratorId(); + openUrl = orchestratorService.openOrchestrator(userName, workspace, orchestratorId, dssLabelList); + OrchestratorVo orchestratorVo = orchestratorService.getOrchestratorVoById(orchestratorId); + LOGGER.info("open url is {}, orcId is {}, dssLabels is {}", openUrl, orchestratorId, dssLabelList); + return Message.ok().data("OrchestratorOpenUrl", openUrl). + data("OrchestratorVo", orchestratorVo); + } + + /** + * 获取编排模式下的所有版本号 + * + * @param queryOrchestratorVersion + * @return + * @throws Exception + */ + @RequestMapping(path = "getVersionByOrchestratorId", method = RequestMethod.POST) + public Message getVersionByOrchestratorId(@RequestBody QueryOrchestratorVersion queryOrchestratorVersion) throws Exception { + List list = orchestratorService.getVersionByOrchestratorId(queryOrchestratorVersion.getOrchestratorId()); + return Message.ok().data("list", list); + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java new file mode 100644 index 000000000..e939b900a --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.service; + +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorCreateRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorDeleteRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.CommonOrchestratorVo; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + + +public interface OrchestratorFrameworkService { + + CommonOrchestratorVo createOrchestrator(String username, OrchestratorCreateRequest orchestratorCreateRequest, Workspace workspace) throws Exception; + + CommonOrchestratorVo modifyOrchestrator(String username, OrchestratorModifyRequest orchestratorModifyRequest, Workspace workspace) throws Exception; + + CommonOrchestratorVo deleteOrchestrator(String username, OrchestratorDeleteRequest orchestratorDeleteRequest, Workspace workspace) throws Exception; + +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java new file mode 100644 index 000000000..f32984959 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.service; + +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; + + +public interface OrchestratorPluginService { + + ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertOrchestration requestConversionOrchestration); + + ResponseConvertOrchestrator getConvertOrchestrationStatus(String id); + +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java new file mode 100644 index 000000000..a26fd2830 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java @@ -0,0 +1,122 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.service; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestOrchestratorInfos; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOrchestratorInfos; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; + + +public interface OrchestratorService { + /** + * 新建编排,实例化一个用户的Orchestrator,并创建数据库记录 + * + * @param dssOrchestratorInfo + * @return + */ + OrchestratorVo createOrchestrator(String userName, + Workspace workspace, + String projectName, + Long projectId, + String description, + DSSOrchestratorInfo dssOrchestratorInfo, + List dssLabels) throws Exception; + + + /** + * 更新编排,更新编排的基本信息 + * + * @param dssOrchestratorInfo + */ + void updateOrchestrator(String userName, + Workspace workspace, + DSSOrchestratorInfo dssOrchestratorInfo, + List dssLabels) throws Exception; + + /** + * 删除编排,根据编排ID删除一个编排 + * + * @param orchestratorInfoId + */ + void deleteOrchestrator(String userName, + Workspace workspace, + String projectName, + Long orchestratorInfoId, + List dssLabels) throws Exception; + + + /** + * 返回一个编排,包含编排的基本信息和最新版本信息 + * + * @param orchestratorId + * @return + */ + OrchestratorVo getOrchestratorVoById(Long orchestratorId); + + /** + * 获取一个指定版本编排 + * + * @param orchestratorId 編排id + * @param orcVersionId 编排版本id + * @return 编排 + */ + OrchestratorVo getOrchestratorVoByIdAndOrcVersionId(Long orchestratorId, Long orcVersionId); + + /** + * 根据一个集合查找 + * + * @param orchestratorIds + * @return + */ + + List getOrchestratorVoList(List orchestratorIds); + + String openOrchestrator(String userName, Workspace workspace, Long orchestratorId, List dssLabels) throws Exception; + + /** + * 获取编排模式下的版本号 + * + * @param orchestratorId + * @return + */ + List getVersionByOrchestratorId(Long orchestratorId); + + List getOrchestratorVersions(String username, Long projectId, Long orchestratorId); + + String rollbackOrchestrator(String username, Long projectId, String projectName, + Long orchestratorId, String version, DSSLabel dssLabel, Workspace workspace) throws Exception; + + //**** new method + void isExistSameNameBeforeCreate(Long workspaceId, Long projectId, String orchestratorName) throws DSSFrameworkErrorException; + + Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest) throws DSSFrameworkErrorException; + + List getListByPage(OrchestratorRequest orchestratorRequest, String username); + + ResponseOrchestratorInfos queryOrchestratorInfos(RequestOrchestratorInfos requestOrchestratorInfos); +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java new file mode 100644 index 000000000..7d5c7701c --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java @@ -0,0 +1,296 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.service.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationCreationOperation; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationDeletionOperation; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationService; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationUpdateOperation; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.DSSOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationResponseRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationUpdateRequestRef; +import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.RefOrchestrationContentRequestRef; +import com.webank.wedatasphere.dss.appconn.scheduler.utils.OrchestrationOperationUtils; +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.protocol.project.*; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorRefOrchestration; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.exception.DSSOrchestratorErrorException; +import com.webank.wedatasphere.dss.orchestrator.core.type.DSSOrchestratorRelation; +import com.webank.wedatasphere.dss.orchestrator.core.type.DSSOrchestratorRelationManager; +import com.webank.wedatasphere.dss.orchestrator.core.utils.OrchestratorUtils; +import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; +import com.webank.wedatasphere.dss.orchestrator.loader.LinkedAppConnResolver; +import com.webank.wedatasphere.dss.orchestrator.loader.OrchestratorManager; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorCreateRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorDeleteRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.CommonOrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorFrameworkService; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import org.apache.commons.collections.CollectionUtils; +import org.apache.linkis.protocol.util.ImmutablePair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + + +public class OrchestratorFrameworkServiceImpl implements OrchestratorFrameworkService { + + protected final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + @Autowired + private OrchestratorMapper orchestratorMapper; + @Autowired + private OrchestratorService newOrchestratorService; + @Autowired + private OrchestratorManager orchestratorManager; + @Autowired + private LinkedAppConnResolver linkedAppConnResolver; + /** + * 1.拿到的dss orchestrator的appconn + * 2.然后创建 + * + * @param orchestratorCreateRequest 创建参数 + * @return + */ + @Override + @SuppressWarnings("ConstantConditions") + public CommonOrchestratorVo createOrchestrator(String username, OrchestratorCreateRequest orchestratorCreateRequest, + Workspace workspace) throws Exception { + //是否存在相同的编排名称 + newOrchestratorService.isExistSameNameBeforeCreate(orchestratorCreateRequest.getWorkspaceId(), orchestratorCreateRequest.getProjectId(), orchestratorCreateRequest.getOrchestratorName()); + //判断工程是否存在,并且取出工程名称和空间名称 + DSSProject dssProject = validateOperation(orchestratorCreateRequest.getProjectId(), username); + //1.创建编排实体bean + DSSOrchestratorRelation dssOrchestratorRelation = DSSOrchestratorRelationManager.getDSSOrchestratorRelationByMode(orchestratorCreateRequest.getOrchestratorMode()); + DSSOrchestratorInfo dssOrchestratorInfo = new DSSOrchestratorInfo(); + dssOrchestratorInfo.setType(dssOrchestratorRelation.getDSSOrchestratorName()); + dssOrchestratorInfo.setAppConnName(dssOrchestratorRelation.getBindingAppConnName()); + dssOrchestratorInfo.setDesc(orchestratorCreateRequest.getDescription()); + dssOrchestratorInfo.setCreateTime(new Date()); + dssOrchestratorInfo.setName(orchestratorCreateRequest.getOrchestratorName()); + dssOrchestratorInfo.setCreator(username); + dssOrchestratorInfo.setProjectId(orchestratorCreateRequest.getProjectId()); + dssOrchestratorInfo.setComment(orchestratorCreateRequest.getDescription()); + dssOrchestratorInfo.setSecondaryType(orchestratorCreateRequest.getOrchestratorWays().toString()); + dssOrchestratorInfo.setUses(orchestratorCreateRequest.getUses()); + //new field + dssOrchestratorInfo.setWorkspaceId(workspace.getWorkspaceId()); + dssOrchestratorInfo.setOrchestratorWay(OrchestratorUtils.getModeStr(orchestratorCreateRequest.getOrchestratorWays())); + dssOrchestratorInfo.setOrchestratorMode(orchestratorCreateRequest.getOrchestratorMode()); + dssOrchestratorInfo.setOrchestratorLevel(orchestratorCreateRequest.getOrchestratorLevel()); + //1.去orchestratorFramework创建编排模式 + LOGGER.info("{} begins to create a orchestrator {}.", username, orchestratorCreateRequest); + List dssLabels = Collections.singletonList(new EnvDSSLabel(orchestratorCreateRequest.getLabels().getRoute())); + //2.如果调度系统要求同步创建工作流,向调度系统发送创建工作流的请求 + OrchestrationResponseRef orchestrationResponseRef = tryOrchestrationOperation(dssLabels, true, username, + dssProject.getName(), workspace, dssOrchestratorInfo, + OrchestrationService::getOrchestrationCreationOperation, + (structureOperation, structureRequestRef) -> ((OrchestrationCreationOperation) structureOperation) + .createOrchestration((DSSOrchestrationContentRequestRef) structureRequestRef), "create"); + OrchestratorVo orchestratorVo = newOrchestratorService.createOrchestrator(username, workspace, dssProject.getName(), + dssOrchestratorInfo.getProjectId(), dssOrchestratorInfo.getDesc(), dssOrchestratorInfo, dssLabels); + Long orchestratorId = orchestratorVo.getDssOrchestratorInfo().getId(); + Long orchestratorVersionId = orchestratorVo.getDssOrchestratorVersion().getId(); + LOGGER.info("created orchestration {} with orchestratorId is {}, and versionId is {}.", orchestratorCreateRequest.getOrchestratorName(), orchestratorId, orchestratorVersionId); + //4.将工程和orchestrator的关系存储到的数据库中 + if (orchestrationResponseRef != null) { + Long refProjectId = (Long) orchestrationResponseRef.toMap().get("refProjectId"); + orchestratorMapper.addOrchestratorRefOrchestration(new DSSOrchestratorRefOrchestration(orchestratorId, refProjectId, orchestrationResponseRef.getRefOrchestrationId())); + } + CommonOrchestratorVo commonOrchestratorVo = new CommonOrchestratorVo(); + commonOrchestratorVo.setOrchestratorId(orchestratorId); + return commonOrchestratorVo; + } + + private V tryOrchestrationOperation(List dssLabels, Boolean askProjectSender, String userName, String projectName, + Workspace workspace, DSSOrchestratorInfo dssOrchestrator, + Function getOrchestrationOperation, + BiFunction responseRefConsumer, String operationName) { + ImmutablePair orchestrationPair = getOrchestrationService(dssOrchestrator, userName, workspace, dssLabels); + Long refProjectId, refOrchestrationId; + if (askProjectSender) { + ProjectRefIdResponse projectRefIdResponse = (ProjectRefIdResponse) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() + .ask(new ProjectRefIdRequest(Optional.ofNullable(orchestrationPair).map(ImmutablePair::getValue).map(AppInstance::getId).orElse(null), dssOrchestrator.getProjectId())); + refProjectId = projectRefIdResponse.getRefProjectId(); + refOrchestrationId = null; + } else { + DSSOrchestratorRefOrchestration refOrchestration = orchestratorMapper.getRefOrchestrationId(dssOrchestrator.getId()); + if (refOrchestration != null) { + refProjectId = refOrchestration.getRefProjectId(); + refOrchestrationId = refOrchestration.getRefOrchestrationId(); + } else { + refProjectId = null; + refOrchestrationId = null; + } + } + V orchestrationResponseRef = null; + if (refProjectId != null && orchestrationPair.getKey() != null) { + LOGGER.info("project {} try to {} a orchestration {} in SchedulerAppConn.", projectName, operationName, dssOrchestrator.getName()); + //这里无需进行重名判断,因为只在 SchedulerAppConn进行创建,一旦创建失败,会触发整个编排创建失败。 + orchestrationResponseRef = OrchestrationOperationUtils.tryOrchestrationOperation(orchestrationPair::getKey, getOrchestrationOperation, + dssOrchestrationContentRequestRef -> + dssOrchestrationContentRequestRef.setDSSOrchestration(dssOrchestrator).setProjectName(projectName).setRefProjectId(refProjectId), + refOrchestrationContentRequestRef -> + refOrchestrationContentRequestRef.setRefOrchestrationId(refOrchestrationId).setOrchestrationName(dssOrchestrator.getName()) + .setRefProjectId(refProjectId).setProjectName(projectName), + (structureOperation, structureRequestRef) -> { + structureRequestRef.setDSSLabels(dssLabels).setWorkspace(workspace).setUserName(userName); + return responseRefConsumer.apply(structureOperation, (K) structureRequestRef); + }, operationName + " orchestration " + dssOrchestrator.getName() + " in SchedulerAppConn"); + if (askProjectSender) { + orchestrationResponseRef.toMap().put("refProjectId", refProjectId); + } + } + return orchestrationResponseRef; + } + + @Override + public CommonOrchestratorVo modifyOrchestrator(String username, OrchestratorModifyRequest orchestratorModifyRequest, Workspace workspace) throws Exception { + //判断工程是否存在,并且取出工程名称和空间名称 + DSSProject dssProject = validateOperation(orchestratorModifyRequest.getProjectId(), username); + workspace.setWorkspaceName(dssProject.getWorkspaceName()); + //是否存在相同的编排名称 //todo 返回orchestratorInfo而不是id + Long orchestratorId = newOrchestratorService.isExistSameNameBeforeUpdate(orchestratorModifyRequest); + LOGGER.info("{} begins to update a orchestrator {}.", username, orchestratorModifyRequest.getOrchestratorName()); + List dssLabels = Collections.singletonList(new EnvDSSLabel(orchestratorModifyRequest.getLabels().getRoute())); + DSSOrchestratorRelation dssOrchestratorRelation = DSSOrchestratorRelationManager.getDSSOrchestratorRelationByMode(orchestratorModifyRequest.getOrchestratorMode()); + DSSOrchestratorInfo dssOrchestratorInfo = new DSSOrchestratorInfo(); + dssOrchestratorInfo.setId(orchestratorId); + dssOrchestratorInfo.setType(dssOrchestratorRelation.getDSSOrchestratorName()); + dssOrchestratorInfo.setDesc(orchestratorModifyRequest.getDescription()); + dssOrchestratorInfo.setUpdateTime(new Date(System.currentTimeMillis())); + dssOrchestratorInfo.setAppConnName(dssOrchestratorRelation.getBindingAppConnName()); + dssOrchestratorInfo.setName(orchestratorModifyRequest.getOrchestratorName()); + dssOrchestratorInfo.setCreator(username); + dssOrchestratorInfo.setProjectId(orchestratorModifyRequest.getProjectId()); + dssOrchestratorInfo.setComment(orchestratorModifyRequest.getDescription()); + dssOrchestratorInfo.setSecondaryType(orchestratorModifyRequest.getOrchestratorWays().toString()); + dssOrchestratorInfo.setUpdateUser(username); + dssOrchestratorInfo.setOrchestratorMode(orchestratorModifyRequest.getOrchestratorMode()); + dssOrchestratorInfo.setOrchestratorWay(OrchestratorUtils.getModeStr(orchestratorModifyRequest.getOrchestratorWays())); + dssOrchestratorInfo.setOrchestratorLevel(orchestratorModifyRequest.getOrchestratorLevel()); + dssOrchestratorInfo.setUses(orchestratorModifyRequest.getUses()); + //1.如果调度系统要求同步创建工作流,向调度系统发送更新工作流的请求 + tryOrchestrationOperation(dssLabels, false, username, dssProject.getName(), workspace, dssOrchestratorInfo, + OrchestrationService::getOrchestrationUpdateOperation, + (structureOperation, structureRequestRef) -> ((OrchestrationUpdateOperation) structureOperation) + .updateOrchestration((OrchestrationUpdateRequestRef) structureRequestRef), "update"); + newOrchestratorService.updateOrchestrator(username, workspace, dssOrchestratorInfo, dssLabels); + //3.将工程和orchestrator的关系存储到的数据库中 + CommonOrchestratorVo orchestratorVo = new CommonOrchestratorVo(); + orchestratorVo.setOrchestratorId(orchestratorId); + return orchestratorVo; + } + + @Override + public CommonOrchestratorVo deleteOrchestrator(String username, OrchestratorDeleteRequest orchestratorDeleteRequest, Workspace workspace) throws Exception { + //判断工程是否存在,并且取出工程名称和空间名称 + DSSProject dssProject = validateOperation(orchestratorDeleteRequest.getProjectId(), username); + DSSOrchestratorInfo orchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorDeleteRequest.getId()); + LOGGER.info("{} begins to delete a orchestrator {}.", username, orchestratorInfo.getName()); + List dssLabels = Collections.singletonList(new EnvDSSLabel(orchestratorDeleteRequest.getLabels().getRoute())); + tryOrchestrationOperation(dssLabels, false, username, dssProject.getName(), workspace, orchestratorInfo, + OrchestrationService::getOrchestrationDeletionOperation, + (structureOperation, structureRequestRef) -> ((OrchestrationDeletionOperation) structureOperation) + .deleteOrchestration((RefOrchestrationContentRequestRef) structureRequestRef), "delete"); + + newOrchestratorService.deleteOrchestrator(username, workspace, dssProject.getName(), orchestratorInfo.getId(), dssLabels); + LOGGER.info("delete orchestrator {} by orchestrator framework succeed.", orchestratorInfo.getName()); + CommonOrchestratorVo orchestratorVo = new CommonOrchestratorVo(); + return orchestratorVo; + } + + protected ImmutablePair getOrchestrationService(DSSOrchestratorInfo dssOrchestratorInfo, + String user, + Workspace workspace, + List dssLabels) { + DSSOrchestrator dssOrchestrator = orchestratorManager.getOrCreateOrchestrator(user, workspace.getWorkspaceName(), dssOrchestratorInfo.getType(), dssLabels); + if (CollectionUtils.isEmpty(dssOrchestratorInfo.getLinkedAppConnNames())) { + dssOrchestratorInfo.setLinkedAppConnNames(dssOrchestrator.getLinkedAppConn().stream().map(appConn -> appConn.getAppDesc().getAppName()).collect(Collectors.toList())); + } + SchedulerAppConn appConn = dssOrchestrator.getSchedulerAppConn(); +// if (appConn == null) { +// throw new ExternalOperationWarnException(50322, "DSSOrchestrator " + dssOrchestrator.getName() + " has no SchedulerAppConn."); +// } + if (appConn != null) { + AppInstance appInstance = appConn.getAppDesc().getAppInstances().get(0); + return new ImmutablePair<>(appConn.getOrCreateStructureStandard().getOrchestrationService(appInstance), appInstance); + } else { + return new ImmutablePair<>(null, null); + } + + } + + /** + * Determine whether the project exists. + * and validate user edit_auth for the project. + * + * @param projectId + * @return + * @throws DSSOrchestratorErrorException + */ + private DSSProject validateOperation(long projectId, String username) throws DSSOrchestratorErrorException { + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(projectId); + DSSProject dssProject = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() + .ask(projectInfoRequest); + if (dssProject == null) { + DSSExceptionUtils.dealErrorException(6003, "工程不存在", DSSOrchestratorErrorException.class); + } + if (!hasProjectEditPriv(projectId, username)) { + DSSExceptionUtils.dealErrorException(6004, "用户没有工程编辑权限", DSSOrchestratorErrorException.class); + } + return dssProject; + } + + private boolean hasProjectEditPriv(Long projectId, String username) { + ProjectUserAuthResponse projectUserAuthResponse = (ProjectUserAuthResponse) DSSSenderServiceFactory.getOrCreateServiceInstance() + .getProjectServerSender().ask(new ProjectUserAuthRequest(projectId, username)); + boolean hasEditPriv = false; + if (!CollectionUtils.isEmpty(projectUserAuthResponse.getPrivList())) { + hasEditPriv = projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_EDIT.getRank()) || + projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_RELEASE.getRank()); + } + return hasEditPriv || projectUserAuthResponse.getProjectOwner().equals(username); + } + +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java new file mode 100644 index 000000000..523fce553 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java @@ -0,0 +1,226 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.service.impl; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.DSSLabelUtil; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorRefOrchestration; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext; +import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorJobMapper; +import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; +import com.webank.wedatasphere.dss.orchestrator.publish.ExportDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.publish.job.ConversionJobEntity; +import com.webank.wedatasphere.dss.orchestrator.publish.job.OrchestratorConversionJob; +import com.webank.wedatasphere.dss.orchestrator.server.constant.DSSOrchestratorConstant; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorPluginService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.commons.collections.CollectionUtils; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + + +public class OrchestratorPluginServiceImpl implements OrchestratorPluginService { + + private final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + @Autowired + private OrchestratorMapper orchestratorMapper; + + @Autowired + private OrchestratorJobMapper orchestratorJobMapper; + + @Autowired + private DSSOrchestratorContext dssOrchestratorContext; + + private ExecutorService releaseThreadPool = Utils.newCachedThreadPool(50, "Convert-Orchestration-Thread-", true); + private AtomicInteger idGenerator = new AtomicInteger(); + + @Override + public ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertOrchestration requestConversionOrchestration) { + LOGGER.info("conversion request is called, request is {}.", DSSCommonUtils.COMMON_GSON.toJson(requestConversionOrchestration)); + Long toPublishOrcId; + if(requestConversionOrchestration.getOrcAppId() != null) { + OrchestratorInfo orchestratorInfo = orchestratorMapper.getOrcInfoByAppId(requestConversionOrchestration.getOrcAppId()); + toPublishOrcId = orchestratorInfo.getOrchestratorId(); + } else if (CollectionUtils.isNotEmpty(requestConversionOrchestration.getOrcIds())) { + toPublishOrcId = requestConversionOrchestration.getOrcIds().get(0); + } else { + return new ResponseConvertOrchestrator("-1", ResponseOperateOrchestrator.failed("Both orcAppId and orcIds are not exists.")); + } + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(toPublishOrcId); + long projectId = dssOrchestratorInfo.getProjectId(); + //这里的 key 为 DSS 具体编排(如 DSS 工作流)的 id;这里的 value 为 DSS 编排所对应的第三方调度系统的工作流 ID + //请注意:由于对接的 SchedulerAppConn 调度系统有可能没有实现 OrchestrationService, + //所以可能存在 DSS 在创建 DSS 编排时,无法同步去 SchedulerAppConn 创建工作流的情况,从而导致这个 Map 的所有 value 都为 null。 + Map orchestrationIdMap = new HashMap<>(); + List publishedOrcIds; + List labels = LabelBuilderFactoryContext.getLabelBuilderFactory().getLabels(requestConversionOrchestration.getLabels()); + if(requestConversionOrchestration.isConvertAllOrcs()) { + //这个地方应该是要获取所有的已经发布过的orchestrator + publishedOrcIds = orchestratorMapper.getAllOrcIdsByProjectId(projectId); + if (!publishedOrcIds.contains(toPublishOrcId)) { + publishedOrcIds.add(toPublishOrcId); + } + for (Long orcId : publishedOrcIds) { + int validFlag = 1; + if (orcId.longValue() == toPublishOrcId.longValue() && !DSSLabelUtil.isDevEnv(labels)) { + validFlag = 0; + } + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orcId, validFlag); + if (dssOrchestratorVersion == null) { + continue; + } + DSSOrchestratorRefOrchestration dssOrchestratorRefOrchestration = orchestratorMapper.getRefOrchestrationId(orcId); + if(dssOrchestratorRefOrchestration != null) { + orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), dssOrchestratorRefOrchestration.getRefOrchestrationId()); + } else { + orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), null); + } + } + } else { + publishedOrcIds = new ArrayList<>(); + if(requestConversionOrchestration.getOrcAppId() != null) { + publishedOrcIds.add(toPublishOrcId); + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(toPublishOrcId,1); + if (dssOrchestratorVersion != null) { + DSSOrchestratorRefOrchestration dssOrchestratorRefOrchestration = orchestratorMapper.getRefOrchestrationId(toPublishOrcId); + if(dssOrchestratorRefOrchestration != null) { + orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), dssOrchestratorRefOrchestration.getRefOrchestrationId()); + } else { + orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), null); + } + } + } + if(requestConversionOrchestration.getOrcIds() != null && !requestConversionOrchestration.getOrcIds().isEmpty()) { + for (Long orcId : requestConversionOrchestration.getOrcIds()) { + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orcId,1); + if (dssOrchestratorVersion == null) { + continue; + } + publishedOrcIds.add(orcId); + DSSOrchestratorRefOrchestration dssOrchestratorRefOrchestration = orchestratorMapper.getRefOrchestrationId(orcId); + if(dssOrchestratorRefOrchestration != null) { + orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), dssOrchestratorRefOrchestration.getRefOrchestrationId()); + } else { + orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), null); + } + } + } + } + if(orchestrationIdMap.isEmpty()) { + LOGGER.info("the project {} has no suitable workflow, the publish by user {} is ignored.", projectId, + requestConversionOrchestration.getUserName()); + return new ResponseConvertOrchestrator("no-necessary-id", ResponseOperateOrchestrator.failed("No suitable workflow(s) found, publish is ignored.")); + } + OrchestratorConversionJob job = new OrchestratorConversionJob(); + job.setId(generateId()); + LOGGER.info("user {} try to submit a conversion job {}, the orchestrationIdMap is {}, the orcIdList is {}.", requestConversionOrchestration.getUserName(), + job.getId(), orchestrationIdMap, publishedOrcIds); + ConversionJobEntity entity = new ConversionJobEntity(); + entity.setResponse(ResponseOperateOrchestrator.inited()); + entity.setCreateTime(new Date()); + entity.setUserName(requestConversionOrchestration.getUserName()); + entity.setOrchestrationIdMap(orchestrationIdMap); + entity.setOrcIdList(publishedOrcIds); + entity.setLabels(labels); + entity.setWorkspace((Workspace) requestConversionOrchestration.getWorkspace()); + DSSProject dssProject = new DSSProject(); + dssProject.setId(projectId); + entity.setProject(dssProject); + job.setConversionJobEntity(entity); + job.setConversionDSSOrchestratorPlugins(dssOrchestratorContext.getOrchestratorPlugins()); + job.afterConversion(response -> this.updateDBAfterConversion(toPublishOrcId, response, job, requestConversionOrchestration)); + //submit it + releaseThreadPool.submit(job); + DSSOrchestratorConstant.orchestratorConversionJobMap.put(job.getId(), job); + // todo insert publishJob + return new ResponseConvertOrchestrator(job.getId(), entity.getResponse()); + } + + @Override + public ResponseConvertOrchestrator getConvertOrchestrationStatus(String id) { + OrchestratorConversionJob job = DSSOrchestratorConstant.orchestratorConversionJobMap.get(id); + // todo 找不到从db加载 + if(job.getConversionJobEntity().getResponse().isCompleted()) { + DSSOrchestratorConstant.orchestratorConversionJobMap.remove(job.getId()); + } + return new ResponseConvertOrchestrator(job.getId(), job.getConversionJobEntity().getResponse()); + } + + private String generateId() { + return String.valueOf(idGenerator.getAndIncrement()); + } + + @Transactional(rollbackFor = Exception.class) + private void updateDBAfterConversion(Long toPublishOrcId,ResponseOperateOrchestrator response, + OrchestratorConversionJob job, + RequestFrameworkConvertOrchestration requestConversionOrchestration) { + LOGGER.info("{} completed with status {}.", job.getId(), response.getJobStatus()); + if (response.isSucceed()) { + //1. 进行导出,用于升级版本,目的是为了复用原来的代码 + List orcIdList = job.getConversionJobEntity().getOrcIdList(); + LOGGER.info("the orcIdList is:{}", orcIdList); + // 开发环境才新增版本号 + if(DSSLabelUtil.isDevEnv(job.getConversionJobEntity().getLabels())){ + orcIdList.forEach(DSSExceptionUtils.handling(orcId -> { + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orcId); + dssOrchestratorContext.getDSSOrchestratorPlugin(ExportDSSOrchestratorPlugin.class) + .orchestratorVersionIncrease(orcId, job.getConversionJobEntity().getUserName(), requestConversionOrchestration.getComment(), + (Workspace) requestConversionOrchestration.getWorkspace(), dssOrchestratorInfo, + job.getConversionJobEntity().getProject().getName(), job.getConversionJobEntity().getLabels()); + })); + } + //2. 做一个标记表示已经发布过了 + orcIdList.forEach(orchestratorMapper::setPublished); + //3.更新当前的提交的发布编排模式版本的common + updateCurrentPublishOrchestratorCommon(toPublishOrcId,requestConversionOrchestration.getComment(),job.getConversionJobEntity().getLabels()); + } + } + + public void updateCurrentPublishOrchestratorCommon(Long orcId,String realComment,List dssLabels) { + //DEV环境获取获取最新有效的版本即可 + int validFlag = 1; + if (!DSSLabelUtil.isDevEnv(dssLabels)) { + //PROD获取当前编排模式最新版本的ID + validFlag = 0; + } + DSSOrchestratorVersion version = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orcId,validFlag); + DSSOrchestratorVersion updateCommentVersion = new DSSOrchestratorVersion(); + updateCommentVersion.setId(version.getId()); + updateCommentVersion.setComment(realComment); + updateCommentVersion.setUpdateTime(new Date()); + updateCommentVersion.setValidFlag(1); + orchestratorMapper.updateOrchestratorVersion(updateCommentVersion); + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java new file mode 100644 index 000000000..ba1ec1d4b --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java @@ -0,0 +1,438 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.service.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.DSSLabelUtil; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthRequest; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthResponse; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestOrchestratorInfos; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestProjectUpdateOrcVersion; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOrchestratorInfos; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.exception.DSSOrchestratorErrorException; +import com.webank.wedatasphere.dss.orchestrator.core.utils.OrchestratorUtils; +import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; +import com.webank.wedatasphere.dss.orchestrator.loader.OrchestratorManager; +import com.webank.wedatasphere.dss.orchestrator.publish.utils.OrchestrationDevelopmentOperationUtils; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.*; +import com.webank.wedatasphere.dss.standard.app.development.ref.*; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.service.RefCRUDService; +import com.webank.wedatasphere.dss.standard.app.development.service.RefQueryService; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +@Service +public class OrchestratorServiceImpl implements OrchestratorService { + + private static final Logger LOGGER = LoggerFactory.getLogger(OrchestratorServiceImpl.class); + @Autowired + private OrchestratorManager orchestratorManager; + @Autowired + private OrchestratorMapper orchestratorMapper; + @Autowired + private ContextService contextService; + + private static final int VALID_FLAG = 1; + + @Override + @Transactional(rollbackFor = Exception.class) + public OrchestratorVo createOrchestrator(String userName, + Workspace workspace, + String projectName, + Long projectId, + String description, + DSSOrchestratorInfo dssOrchestratorInfo, + List dssLabels) throws Exception { + OrchestratorVo orchestratorVo = new OrchestratorVo(); + //todo 增加校验 + String uuid = UUID.randomUUID().toString(); + + //作为Orchestrator的唯一标识,包括跨环境导入导出也不发生变化。 + dssOrchestratorInfo.setUUID(uuid); + + String version = OrchestratorUtils.generateNewVersion(); + String contextId = contextService.createContextID(workspace.getWorkspaceName(), projectName, dssOrchestratorInfo.getName(), version, userName); + LOGGER.info("Create a new ContextId: {} for new orchestrator {}.", contextId, dssOrchestratorInfo.getName()); + //1. 访问DSS工作流微模块创建工作流 + RefJobContentResponseRef appRef = tryRefOperation(dssOrchestratorInfo, userName, workspace, dssLabels, null, + developmentService -> ((RefCRUDService) developmentService).getRefCreationOperation(), + dssContextRequestRef -> dssContextRequestRef.setContextId(contextId), + projectRefRequestRef -> projectRefRequestRef.setProjectName(projectName).setRefProjectId(projectId), + (developmentOperation, developmentRequestRef) -> { + DSSOrchestrator dssOrchestrator = orchestratorManager.getOrCreateOrchestrator(userName, + workspace.getWorkspaceName(), dssOrchestratorInfo.getType(), dssLabels); + Map dssJobContent = MapUtils.newCommonMapBuilder() + .put(OrchestratorRefConstant.DSS_ORCHESTRATOR_INFO_KEY, dssOrchestratorInfo) + .put(OrchestratorRefConstant.ORCHESTRATOR_VERSION_KEY, version) + .put(OrchestratorRefConstant.ORCHESTRATION_SCHEDULER_APP_CONN, Optional.ofNullable(dssOrchestrator) + .map(DSSOrchestrator::getSchedulerAppConn).map(AppConn::getAppDesc).map(AppDesc::getAppName) + .map(Object::toString).orElse("NULL")).build(); + DSSJobContentRequestRef requestRef = (DSSJobContentRequestRef) developmentRequestRef; + requestRef.setDSSJobContent(dssJobContent); + return ((RefCreationOperation) developmentOperation).createRef(requestRef); + }, "create"); + + //2.往数据库插入一条DSS编排模式 + orchestratorMapper.addOrchestrator(dssOrchestratorInfo); + + DSSOrchestratorVersion dssOrchestratorVersion = new DSSOrchestratorVersion(); + dssOrchestratorVersion.setOrchestratorId(dssOrchestratorInfo.getId()); + dssOrchestratorVersion.setAppId((Long) appRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY)); + dssOrchestratorVersion.setContent((String) appRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY)); + dssOrchestratorVersion.setComment(description); + dssOrchestratorVersion.setProjectId(projectId); + dssOrchestratorVersion.setSource("Orchestrator create"); + dssOrchestratorVersion.setUpdater(userName); + dssOrchestratorVersion.setVersion(version); + dssOrchestratorVersion.setUpdateTime(new Date()); + dssOrchestratorVersion.setValidFlag(VALID_FLAG); + dssOrchestratorVersion.setFormatContextId(contextId); + // 4. 持久化编排的版本信息 + orchestratorMapper.addOrchestratorVersion(dssOrchestratorVersion); + orchestratorVo.setDssOrchestratorInfo(dssOrchestratorInfo); + orchestratorVo.setDssOrchestratorVersion(dssOrchestratorVersion); + + return orchestratorVo; + } + + + private V tryRefOperation(DSSOrchestratorInfo dssOrchestratorInfo, + String userName, + Workspace workspace, + List dssLabels, + BiFunction getDevelopmentService, + Function getOperation, + Consumer contextRequestRefConsumer, + Consumer projectRefRequestRefConsumer, + BiFunction responseRefConsumer, + String operationName) throws DSSErrorException { + DSSOrchestrator dssOrchestrator = orchestratorManager.getOrCreateOrchestrator(userName, + workspace.getWorkspaceName(), dssOrchestratorInfo.getType(), dssLabels); + dssOrchestratorInfo.setAppConnName(dssOrchestrator.getAppConn().getAppDesc().getAppName()); + if (getDevelopmentService == null) { + getDevelopmentService = DevelopmentIntegrationStandard::getRefCRUDService; + } + return OrchestrationDevelopmentOperationUtils.tryOrchestrationOperation(dssOrchestratorInfo, dssOrchestrator, + userName, workspace, dssLabels, getDevelopmentService, getOperation, contextRequestRefConsumer, projectRefRequestRefConsumer, + responseRefConsumer, operationName); + } + + @Override + public void updateOrchestrator(String userName, + Workspace workspace, + DSSOrchestratorInfo dssOrchestratorInfo, + List dssLabels) throws Exception { + orchestratorMapper.updateOrchestrator(dssOrchestratorInfo); + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(dssOrchestratorInfo.getId(), VALID_FLAG); + + tryRefOperation(dssOrchestratorInfo, userName, workspace, dssLabels, null, + developmentService -> ((RefCRUDService) developmentService).getRefUpdateOperation(), + null, + projectRefRequestRef -> projectRefRequestRef.setRefProjectId(dssOrchestratorInfo.getProjectId()), + (developmentOperation, developmentRequestRef) -> { + UpdateRequestRef requestRef = (UpdateRequestRef) developmentRequestRef; + Map dssJobContent = MapUtils.newCommonMapBuilder() + .put(OrchestratorRefConstant.ORCHESTRATION_NAME, dssOrchestratorInfo.getName()) + .put(OrchestratorRefConstant.ORCHESTRATION_DESCRIPTION, dssOrchestratorInfo.getComment()) + .put(OrchestratorRefConstant.ORCHESTRATION_USES, dssOrchestratorInfo.getUses()).build(); + requestRef.setDSSJobContent(dssJobContent); + requestRef.setRefJobContent(MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, dssOrchestratorVersion.getAppId(), + OrchestratorRefConstant.ORCHESTRATOR_ID_KEY, dssOrchestratorInfo.getId())); + return ((RefUpdateOperation) developmentOperation).updateRef(requestRef); + }, "update"); + } + + @Override + public void deleteOrchestrator(String userName, + Workspace workspace, + String projectName, + Long orchestratorInfoId, + List dssLabels) throws Exception { + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorInfoId); + if (null == dssOrchestratorInfo) { + LOGGER.error("Not exists orchestration {} in project {}.", orchestratorInfoId, projectName); + DSSExceptionUtils.dealErrorException(61123, + String.format("Not exists orchestration %s in project %s.", orchestratorInfoId, projectName), + DSSErrorException.class); + } + orchestratorMapper.getOrchestratorVersions(dssOrchestratorInfo.getProjectId(), orchestratorInfoId).forEach(dssOrchestratorVersion -> { + LOGGER.info("user {} try to delete the DSS project {} orchestration(such as DSS workflow) {} of orchestrator {} in version {}.", + userName, projectName, dssOrchestratorVersion.getAppId(), dssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion()); + try { + tryRefOperation(dssOrchestratorInfo, userName, workspace, dssLabels, null, + developmentService -> ((RefCRUDService) developmentService).getRefDeletionOperation(), + null, projectRefRequestRef -> projectRefRequestRef.setRefProjectId(dssOrchestratorInfo.getProjectId()), + (developmentOperation, developmentRequestRef) -> { + RefJobContentRequestRef requestRef = (RefJobContentRequestRef) developmentRequestRef; + Map refJobContent = MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, dssOrchestratorVersion.getAppId(), + OrchestratorRefConstant.ORCHESTRATOR_ID_KEY, orchestratorInfoId); + requestRef.setRefJobContent(refJobContent); + return ((RefDeletionOperation) developmentOperation).deleteRef(requestRef); + }, "delete"); + } catch (DSSErrorException e) { + throw new ExternalOperationWarnException(e.getErrCode(), e.getMessage(), e); + } + orchestratorMapper.deleteOrchestratorVersion(dssOrchestratorVersion.getId()); + }); + orchestratorMapper.deleteOrchestrator(orchestratorInfoId); + } + + @Override + public List getOrchestratorVoList(List orchestratorIds) { + List orchestratorVoList = new ArrayList<>(); + orchestratorIds.forEach(orchestratorId -> { + OrchestratorVo orchestratorVo = getOrchestratorVoById(orchestratorId); + orchestratorVoList.add(orchestratorVo); + }); + return orchestratorVoList; + } + + @Override + public String openOrchestrator(String userName, Workspace workspace, Long orchestratorId, List dssLabels) throws Exception { + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorId); + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orchestratorId, VALID_FLAG); + if (null == dssOrchestratorInfo || null == dssOrchestratorVersion) { + throw new DSSOrchestratorErrorException(1000856, "can not find orc from db for orcId: " + orchestratorId); + } + QueryJumpUrlResponseRef responseRef = (QueryJumpUrlResponseRef) tryRefOperation(dssOrchestratorInfo, userName, + workspace, dssLabels, DevelopmentIntegrationStandard::getRefQueryService, + developmentService -> ((RefQueryService) developmentService).getRefQueryOperation(), + null, + projectRefRequestRef -> projectRefRequestRef.setRefProjectId(dssOrchestratorInfo.getProjectId()), + (developmentOperation, developmentRequestRef) -> { + RefJobContentRequestRef requestRef = (RefJobContentRequestRef) developmentRequestRef; + requestRef.setRefJobContent(MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, dssOrchestratorVersion.getAppId(), + OrchestratorRefConstant.ORCHESTRATOR_ID_KEY, orchestratorId)); + return ((RefQueryOperation) developmentOperation).query(requestRef); + }, "open"); + + return responseRef.getJumpUrl(); + } + + @Override + public OrchestratorVo getOrchestratorVoById(Long orchestratorId) { + + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorId); + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orchestratorId, VALID_FLAG); + + OrchestratorVo orchestratorVo = new OrchestratorVo(); + orchestratorVo.setDssOrchestratorInfo(dssOrchestratorInfo); + orchestratorVo.setDssOrchestratorVersion(dssOrchestratorVersion); + return orchestratorVo; + } + + @Override + public OrchestratorVo getOrchestratorVoByIdAndOrcVersionId(Long orchestratorId, Long orcVersionId) { + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorId); + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getOrcVersionByIdAndOrcVersionId(orchestratorId, orcVersionId, VALID_FLAG); + + OrchestratorVo orchestratorVo = new OrchestratorVo(); + orchestratorVo.setDssOrchestratorInfo(dssOrchestratorInfo); + orchestratorVo.setDssOrchestratorVersion(dssOrchestratorVersion); + return orchestratorVo; + } + + @Override + public List getVersionByOrchestratorId(Long orchestratorId) { + return orchestratorMapper.getVersionByOrchestratorId(orchestratorId); + } + + @Override + public List getOrchestratorVersions(String username, Long projectId, Long orchestratorId) { + LOGGER.info("user {} wants to get orc versions in projectId {} for orcId {}", username, projectId, orchestratorId); + List orchestratorVersions = orchestratorMapper.getOrchestratorVersions(projectId, orchestratorId); + LOGGER.info("projectId is {} , orcId is {}, orcVersions are {}", projectId, orchestratorId, orchestratorVersions); + return orchestratorVersions; + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public String rollbackOrchestrator(String userName, Long projectId, String projectName, + Long orchestratorId, String version, DSSLabel dssLabel, Workspace workspace) throws Exception { + //1.新建一个版本 + //2.然后将version的版本内容进行去workflow进行cp + //3.然后把生产的内容进行update到数据库 + String latestVersion = orchestratorMapper.getLatestVersion(orchestratorId, 1); + List labels = new ArrayList<>(); + labels.add(dssLabel); + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorId); + String newVersion = OrchestratorUtils.increaseVersion(latestVersion); + DSSOrchestratorVersion dssOrchestratorVersion = new DSSOrchestratorVersion(); + String comment = "回滚工作流到版本:" + version; + dssOrchestratorVersion.setOrchestratorId(orchestratorId); + dssOrchestratorVersion.setVersion(newVersion); + dssOrchestratorVersion.setUpdateTime(new Date()); + dssOrchestratorVersion.setProjectId(projectId); + dssOrchestratorVersion.setUpdater(userName); + dssOrchestratorVersion.setComment(comment); + dssOrchestratorVersion.setSource("rollback from version :" + version); + dssOrchestratorVersion.setValidFlag(1); + DSSOrchestratorVersion dbOrcVersion = orchestratorMapper.getVersionByOrchestratorIdAndVersion(orchestratorId, version); + if (dbOrcVersion == null) { + DSSExceptionUtils.dealErrorException(60070, "Rollback version " + version + " is not exist.", DSSOrchestratorErrorException.class); + } + String contextId = contextService.createContextID(workspace.getWorkspaceName(), projectName, dssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion(), userName); + dssOrchestratorVersion.setContextId(contextId); + LOGGER.info("Create a new ContextId {} for rollback the orchestration {} to version {}.", contextId, dssOrchestratorInfo.getName(), version); + RefJobContentResponseRef responseRef = tryRefOperation(dssOrchestratorInfo, userName, workspace, Collections.singletonList(dssLabel), null, + developmentService -> ((RefCRUDService) developmentService).getRefCopyOperation(), + dssContextRequestRef -> dssContextRequestRef.setContextId(contextId), + projectRefRequestRef -> projectRefRequestRef.setRefProjectId(dssOrchestratorInfo.getProjectId()).setProjectName(projectName), + (developmentOperation, developmentRequestRef) -> { + CopyRequestRef copyRequestRef = (CopyRequestRef) developmentRequestRef; + Map refJobContent = MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, dbOrcVersion.getAppId(), + OrchestratorRefConstant.ORCHESTRATION_DESCRIPTION, comment); + copyRequestRef.setNewVersion(dssOrchestratorVersion.getVersion()) + .setRefJobContent(refJobContent); + return ((RefCopyOperation) developmentOperation).copyRef(copyRequestRef); + }, "copy"); + dssOrchestratorVersion.setAppId((Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY)); + dssOrchestratorVersion.setContent((String) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY)); + dssOrchestratorVersion.setFormatContextId(contextId); + //update appConn node contextId + orchestratorMapper.addOrchestratorVersion(dssOrchestratorVersion); +// synProjectOrchestratorVersionId(dssOrchestratorVersion, labels); + return dssOrchestratorVersion.getVersion(); + } + + + public void synProjectOrchestratorVersionId(DSSOrchestratorVersion dssOrchestratorVersion, List dssLabels) { + //Is dev environment + if (DSSLabelUtil.isDevEnv(dssLabels)) { + RequestProjectUpdateOrcVersion updateOrchestratorVersion = new RequestProjectUpdateOrcVersion(); + updateOrchestratorVersion.setOrchestratorId(dssOrchestratorVersion.getOrchestratorId()); + updateOrchestratorVersion.setVersionId(dssOrchestratorVersion.getId()); + updateOrchestratorVersion.setDssLabels(dssLabels); + updateOrchestratorVersion.setProjectId(dssOrchestratorVersion.getProjectId()); + //将最新的编排版本ID更新到工程编排表 + DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() + .ask(updateOrchestratorVersion); + } + } + + //新建前是否存在相同的编排名称 + @Override + public void isExistSameNameBeforeCreate(Long workspaceId, Long projectId, String arrangeName) throws DSSFrameworkErrorException { + List retList = orchestratorMapper.getByNameAndProjectId(projectId, arrangeName); + if (CollectionUtils.isNotEmpty(retList)) { + DSSFrameworkErrorException.dealErrorException(60000, "编排名称已经存在"); + } + } + + //是否存在相同的编排名称,如果不存在相同的编排名称則返回编排id + @Override + public Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest) throws DSSFrameworkErrorException { + DSSOrchestratorInfo orchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorModifRequest.getId()); + if (orchestratorInfo == null) { + DSSFrameworkErrorException.dealErrorException(60000, "编排模式ID=" + orchestratorModifRequest.getId() + "不存在"); + } + //若修改了编排名称,检查是否存在相同的编排名称 + if (!orchestratorModifRequest.getOrchestratorName().equals(orchestratorInfo.getName())) { + isExistSameNameBeforeCreate(orchestratorModifRequest.getWorkspaceId(), orchestratorModifRequest.getProjectId(), orchestratorModifRequest.getOrchestratorName()); + } + return orchestratorInfo.getId(); + } + + /** + * 查询编排模式 + * + * @param orchestratorRequest request of front-end + * @param username username + * @return list of OrchestratorBaseInfo + */ + @Override + public List getListByPage(OrchestratorRequest orchestratorRequest, String username) { + List list = orchestratorMapper.queryOrchestratorInfos(new HashMap() {{ + put("workspace_id", orchestratorRequest.getWorkspaceId()); + put("project_id", orchestratorRequest.getProjectId()); + put("orchestrator_mode", orchestratorRequest.getOrchestratorMode()); + }}); + List retList = new ArrayList<>(list.size()); + if (!CollectionUtils.isEmpty(list)) { + //todo Is used in front-end? + ProjectUserAuthResponse projectUserAuthResponse = (ProjectUserAuthResponse) DSSSenderServiceFactory.getOrCreateServiceInstance() + .getProjectServerSender().ask(new ProjectUserAuthRequest(orchestratorRequest.getProjectId(), username)); + boolean isReleasable = false, isEditable = false; + if (!CollectionUtils.isEmpty(projectUserAuthResponse.getPrivList())) { + isReleasable = projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_RELEASE.getRank()); + isEditable = projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_EDIT.getRank()); + } + isReleasable = isReleasable || projectUserAuthResponse.getProjectOwner().equals(username); + isEditable = isEditable || projectUserAuthResponse.getProjectOwner().equals(username); + + for (DSSOrchestratorInfo dssOrchestratorInfo : list) { + OrchestratorBaseInfo orchestratorBaseInfo = new OrchestratorBaseInfo(); + BeanUtils.copyProperties(dssOrchestratorInfo, orchestratorBaseInfo); + orchestratorBaseInfo.setOrchestratorWays(OrchestratorUtils.convertList(dssOrchestratorInfo.getOrchestratorWay())); + orchestratorBaseInfo.setOrchestratorName(dssOrchestratorInfo.getName()); + orchestratorBaseInfo.setCreateUser(dssOrchestratorInfo.getCreator()); + orchestratorBaseInfo.setOrchestratorId(dssOrchestratorInfo.getId()); + orchestratorBaseInfo.setEditable(isEditable || isReleasable); + orchestratorBaseInfo.setReleasable(isReleasable); + retList.add(orchestratorBaseInfo); + } + } + return retList; + } + + @Override + public ResponseOrchestratorInfos queryOrchestratorInfos(RequestOrchestratorInfos requestOrchestratorInfos) { + List orchestratorInfos = orchestratorMapper.queryOrchestratorInfos(new HashMap() {{ + put("workspace_id", requestOrchestratorInfos.getWorkspaceId()); + put("project_id", requestOrchestratorInfos.getProjectId()); + put("name", requestOrchestratorInfos.getName()); + put("orchestrator_mode", requestOrchestratorInfos.getOrchestratorMode()); + }}); + return new ResponseOrchestratorInfos(orchestratorInfos); + } + +} \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/DSSOrchestratorServerApplication.scala b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/DSSOrchestratorServerApplication.scala new file mode 100644 index 000000000..10455d5a5 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/DSSOrchestratorServerApplication.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + + +object DSSOrchestratorServerApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala new file mode 100644 index 000000000..b26472b07 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.receiver + +import com.webank.wedatasphere.dss.orchestrator.common.protocol._ +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext +import com.webank.wedatasphere.dss.orchestrator.server.service.{OrchestratorPluginService, OrchestratorService} +import javax.annotation.PostConstruct +import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + + +@Component +class DSSOrchestratorChooser extends ReceiverChooser { + + @Autowired + var orchestratorService: OrchestratorService = _ + + @Autowired + var orchestratorPluginService: OrchestratorPluginService = _ + + @Autowired + var orchestratorContext: DSSOrchestratorContext = _ + + var receiver: Option[DSSOrchestratorReceiver] = _ + + @PostConstruct + def init(): Unit = receiver = Some(new DSSOrchestratorReceiver(orchestratorService, orchestratorPluginService, orchestratorContext)) + + override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = event.message match { + case _: RequestExportOrchestrator => receiver + case _: RequestImportOrchestrator => receiver + case _: RequestQueryOrchestrator => receiver + case _: RequestFrameworkConvertOrchestration => receiver + case _: RequestFrameworkConvertOrchestrationStatus => receiver + case _: RequestAddVersionAfterPublish => receiver + case _: RequestOrchestratorVersion => receiver + case _: RequestOrchestratorInfos => receiver + case _: RequestQueryByIdOrchestrator => receiver + case _ => None + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala new file mode 100644 index 000000000..783dca8d0 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.server.receiver + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException +import com.webank.wedatasphere.dss.common.protocol.{ResponseExportOrchestrator, ResponseImportOrchestrator} +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo +import com.webank.wedatasphere.dss.orchestrator.common.protocol._ +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext +import com.webank.wedatasphere.dss.orchestrator.publish.{ExportDSSOrchestratorPlugin, ImportDSSOrchestratorPlugin} +import com.webank.wedatasphere.dss.orchestrator.server.service.{OrchestratorPluginService, OrchestratorService} +import org.apache.linkis.rpc.{Receiver, Sender} + +import java.util +import scala.concurrent.duration.Duration + + +class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestratorPluginService: OrchestratorPluginService, orchestratorContext: DSSOrchestratorContext) extends Receiver { + + override def receive(message: Any, sender: Sender): Unit = {} + + override def receiveAndReply(message: Any, sender: Sender): Any = message match { + + case reqExportOrchestrator: RequestExportOrchestrator => + val dssExportOrcResource: util.Map[String, AnyRef] = orchestratorContext.getDSSOrchestratorPlugin(classOf[ExportDSSOrchestratorPlugin]).exportOrchestrator( + reqExportOrchestrator.getUserName, + reqExportOrchestrator.getOrchestratorId, + reqExportOrchestrator.getOrcVersionId, + reqExportOrchestrator.getProjectName, + reqExportOrchestrator.getDssLabels, + reqExportOrchestrator.getAddOrcVersion, + reqExportOrchestrator.getWorkspace) + ResponseExportOrchestrator(dssExportOrcResource.get("resourceId").toString, + dssExportOrcResource.get("version").toString, dssExportOrcResource.get("orcVersionId").asInstanceOf[Long] + ) + + case requestImportOrchestrator: RequestImportOrchestrator => + val importOrcId = orchestratorContext.getDSSOrchestratorPlugin(classOf[ImportDSSOrchestratorPlugin]).importOrchestrator(requestImportOrchestrator) + ResponseImportOrchestrator(importOrcId) + + case addVersionAfterPublish: RequestAddVersionAfterPublish => + orchestratorContext.getDSSOrchestratorPlugin(classOf[ExportDSSOrchestratorPlugin]).addVersionAfterPublish( + addVersionAfterPublish.getUserName, + addVersionAfterPublish.getWorkspace, + addVersionAfterPublish.getOrchestratorId, + addVersionAfterPublish.getOrcVersionId, + addVersionAfterPublish.getProjectName, + addVersionAfterPublish.getDssLabels, addVersionAfterPublish.getComment) + + case requestQueryOrchestrator: RequestQueryOrchestrator => + val requestIdList = requestQueryOrchestrator.getOrchestratorIds + val queryOrchestratorList: java.util.List[OrchestratorVo] = orchestratorService.getOrchestratorVoList(requestIdList) + new ResponseQueryOrchestrator(queryOrchestratorList) + + case requestConversionOrchestration: RequestFrameworkConvertOrchestration => + //发布调度,请注意 + orchestratorPluginService.convertOrchestration(requestConversionOrchestration) + case requestConversionOrchestrationStatus: RequestFrameworkConvertOrchestrationStatus => + orchestratorPluginService.getConvertOrchestrationStatus(requestConversionOrchestrationStatus.getId) + + case requestOrchestratorVersion: RequestOrchestratorVersion => + val projectId = requestOrchestratorVersion.getProjectId + val username = requestOrchestratorVersion.getUsername + val orchestratorId = requestOrchestratorVersion.getOrchestratorId + val orchestratorVersions = orchestratorService.getOrchestratorVersions(username, projectId, orchestratorId) + new ResponseOrchetratorVersion(projectId, orchestratorId, orchestratorVersions) + + case requestOrchestratorInfos: RequestOrchestratorInfos => + orchestratorService.queryOrchestratorInfos(requestOrchestratorInfos); + + case requestQueryByIdOrchestrator: RequestQueryByIdOrchestrator => { + val orcVersionId = requestQueryByIdOrchestrator.getOrcVersionId + val orchestratorId = requestQueryByIdOrchestrator.getOrchestratorId + if (orchestratorId != null) { + orchestratorService.getOrchestratorVoByIdAndOrcVersionId(orchestratorId, orcVersionId) + } else { + orchestratorService.getOrchestratorVoById(orchestratorId) + } + } + + case _ => throw new DSSErrorException(90000, "Not support message type " + message) + } + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} +} diff --git a/dss-framework/dss-framework-project-server/pom.xml b/dss-framework/dss-framework-project-server/pom.xml new file mode 100644 index 000000000..911c0a2dc --- /dev/null +++ b/dss-framework/dss-framework-project-server/pom.xml @@ -0,0 +1,277 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + + 4.0.0 + + dss-framework-project-server + + + + com.webank.wedatasphere.dss + dss-framework-common + ${dss.version} + + + commons-math3 + org.apache.commons + + + + + + org.apache.commons + commons-math3 + provided + + + + com.webank.wedatasphere.dss + dss-appconn-framework + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-structure-integration-standard + ${dss.version} + + + commons-collections + commons-collections + + + dss-common + com.webank.wedatasphere.dss + + + linkis-common + org.apache.linkis + + + + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + + + commons-collections + commons-collections + + + linkis-common + org.apache.linkis + + + commons-lang3 + org.apache.commons + + + commons-logging + commons-logging + + + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + provided + + + + org.glassfish.jersey.ext + jersey-bean-validation + ${jersey.version} + provided + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + linkis-common + org.apache.linkis + + + + + com.webank.wedatasphere.dss + dss-framework-workspace-server + ${dss.version} + + + linkis-common + org.apache.linkis + + + commons-logging + commons-logging + + + commons-codec + commons-codec + + + commons-lang3 + org.apache.commons + + + json4s-jackson_2.11 + org.json4s + + + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-sender-service + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-framework-admin-service + ${dss.version} + + + linkis-common + org.apache.linkis + + + commons-logging + commons-logging + + + commons-codec + commons-codec + + + commons-lang3 + org.apache.commons + + + json4s-jackson_2.11 + org.json4s + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + src/main/resources + + **/*.properties + **/*.xml + **/*.yml + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/assembly/distribution.xml b/dss-framework/dss-framework-project-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..a506dcb06 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/assembly/distribution.xml @@ -0,0 +1,44 @@ + + + + dss-framework-project-server + + dir + + true + dss-framework-project-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ProjectConf.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ProjectConf.java new file mode 100644 index 000000000..34021996f --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ProjectConf.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.conf; + +import org.apache.linkis.common.conf.CommonVars; + + +public interface ProjectConf { + + CommonVars SUPPORT_ABILITY = CommonVars.apply("wds.dss.framework.project.support.ability", "import,export,publish"); + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ProjectSpringConf.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ProjectSpringConf.java new file mode 100644 index 000000000..9b57855b2 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ProjectSpringConf.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.conf; + +import com.webank.wedatasphere.dss.framework.project.service.DSSFrameworkProjectService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectUserService; +import com.webank.wedatasphere.dss.framework.project.service.impl.DSSFrameworkProjectServiceImpl; +import com.webank.wedatasphere.dss.framework.project.service.impl.DSSProjectUserServiceImpl; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class ProjectSpringConf { + + @Bean + @ConditionalOnMissingBean + public DSSProjectUserService createDSSProjectUserService() { + return new DSSProjectUserServiceImpl(); + } + + @Bean + @ConditionalOnMissingBean + public DSSFrameworkProjectService createDSSFrameworkProjectServiceImpl() { + return new DSSFrameworkProjectServiceImpl(); + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ValidationExceptionMapper.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ValidationExceptionMapper.java new file mode 100644 index 000000000..d3c0ea6a8 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/conf/ValidationExceptionMapper.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.conf; + + +import com.webank.wedatasphere.dss.common.utils.MessageUtils; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.validation.ConstraintViolation; +import javax.validation.ConstraintViolationException; +import javax.validation.ValidationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + + + +@Provider +public class ValidationExceptionMapper implements ExceptionMapper { + + private final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + @Override + public Response toResponse(ValidationException e) { + if (LOGGER.isDebugEnabled()){ + LOGGER.debug("failed to validate request bean", e); + } + StringBuilder strBuilder = new StringBuilder(); + for (ConstraintViolation cv : ((ConstraintViolationException) e).getConstraintViolations()) { + strBuilder.append(cv.getMessage()).append(";"); + } + Message message = Message.error(strBuilder.toString()); + return MessageUtils.messageToResponse(message); + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectServerResponse.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectServerResponse.java new file mode 100644 index 000000000..f47d01011 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectServerResponse.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.contant; + +public enum ProjectServerResponse { + PROJECT_NOT_EDIT_AUTH(6001,"没有修改权限"), + PROJECT_NOT_EDIT_NAME(6002,"不能修改工程名称"), + PROJECT_NOT_EXIST(6003,"工程不存在"), + PROJECT_USER_NOT_IN_WORKSPACE(6004,"只有工作空间用户或管理员才能创建工程"), + PROJECT_IS_NOT_ADMIN(6005,"只有创建人或管理员才能编辑工程") + ; + + ProjectServerResponse(int code, String msg) { + this.code = code; + this.msg = msg; + } + + private int code; + private String msg; + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectSpringConfiguration.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectSpringConfiguration.java new file mode 100644 index 000000000..12f52b7c9 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectSpringConfiguration.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.contant; + +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; +import com.webank.wedatasphere.dss.framework.project.service.impl.DSSProjectServiceImpl; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ProjectSpringConfiguration { + + @Bean + @ConditionalOnMissingBean + public DSSProjectService createProjectService() { + return new DSSProjectServiceImpl(); + } + +} \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java new file mode 100644 index 000000000..c3f9dfc19 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.dao; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.po.DSSProjectPo; +import com.webank.wedatasphere.dss.framework.project.entity.po.ProjectRelationPo; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; +import com.webank.wedatasphere.dss.framework.project.entity.vo.ProjectInfoVo; +import com.webank.wedatasphere.dss.framework.project.entity.vo.QueryProjectVo; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +import java.util.List; + +@Mapper +public interface DSSProjectMapper extends BaseMapper { + + void addProject(DSSProjectPo dssProjectPo); + + @Select("select id from dss_project where `name` = #{projectName}") + Long getProjectIdByName(@Param("projectName") String projectName); + + @Select("select `name` from dss_project where `id` = #{projectId}") + String getProjectNameById(@Param("projectId") Long projectId); + + @Select("select `id` from dss_project where `workspace_id` = #{workspaceId} and visible = #{visible}") + List getProjectIdsByWorkspaceId(@Param("workspaceId") Long workspaceId,@Param("visible")int visible); + + + List getListByParam(ProjectQueryRequest projectRequest); + + /** + * 获取工程详情:工程名称、空间名称 + */ + ProjectInfoVo getProjectInfoById(@Param("id") Long id); + + void saveProjectRelation(List projectRelationPoList); + + @Select("select `appconn_instance_project_id` from dss_appconn_project_relation " + + "where `project_id` = #{dssProjectId} and `appconn_instance_id` = #{appInstanceId}") + Long getAppConnProjectId(@Param("appInstanceId")Long appInstanceId, @Param("dssProjectId")Long dssProjectId); + + @Update("update dss_project set `visible` = 0 where `id` = #{projectId}") + void deleteProject(@Param("projectId")Long projectId); + + List getListForAdmin(ProjectQueryRequest projectRequest); + + @Update("DELETE FROM dss_project where `id` = #{projectId}") + void deleteProjectInfo(@Param("projectId") Long projectId); + + /** + * 获取已删除的工程 + * + */ + List getDeletedProjects(ProjectQueryRequest projectRequest); +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectUserMapper.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectUserMapper.java new file mode 100644 index 000000000..32c618636 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectUserMapper.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectUser; +import org.apache.ibatis.annotations.*; + +import java.util.List; + + +@Mapper +public interface DSSProjectUserMapper extends BaseMapper { + @Insert({ + "" + }) + void insertBatchProjectUser(@Param("list") List projectUserList); + + @Delete("delete from dss_project_user where project_id = #{projectID}") + void deleteAllPriv(@Param("projectID") long projectID); + + @Select("SELECT COUNT(0) FROM dss_workspace_user_role WHERE workspace_id = #{workspaceId} AND username = #{username} AND role_id = #{roleId} ") + Long isAdminByUsername(@Param("workspaceId")Long workspaceId,@Param("username")String username,@Param("roleId")int roleId); + List getUserWorkspaceAdminRole(@Param("workspaceId") Long workspaceId, @Param("username") String username); + + + @Select("SELECT COUNT(0) FROM dss_workspace_user_role WHERE workspace_id = #{workspaceId} AND username = #{username}") + Long isWorkspaceUser(@Param("workspaceId")Long workspaceId,@Param("username")String username); + + List getPrivsByProjectId(@Param("projectId") Long projectId); +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml new file mode 100644 index 000000000..fb1e37b25 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml @@ -0,0 +1,112 @@ + + + + + + + id,`name`,`source`, `workspace_id`,`description`,`org_id`,`visibility`,`is_transfer`,`initial_org_id`, + `username`,`create_time`,`create_by`,`product`,`application_area`,`business` + , `user_id`, `create_by_str`, `update_by_str` + + + + INSERT INTO dss_project () + VALUES + (#{id},#{name},#{source}, #{workspaceId}, #{description},#{orgID},#{visibility},#{isTransfer},#{initialOrgID}, + #{username},#{createTime},#{createBy},#{product},#{applicationArea},#{business}, + #{userID},#{createByStr},#{updateByStr}) + + + + + + + + insert into dss_appconn_project_relation + ( + `project_id`, + `appconn_instance_id`, + `appconn_instance_project_id` + ) + values + + ( + #{item.dssProjectId}, + #{item.appInstanceId}, + #{item.appInstanceProjectId} + ) + + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectUserMapper.xml b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectUserMapper.xml new file mode 100644 index 000000000..7c6825c38 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectUserMapper.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectDO.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectDO.java new file mode 100644 index 000000000..0b900239d --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectDO.java @@ -0,0 +1,355 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.io.Serializable; +import java.util.Date; + + + +@TableName(value = "dss_project") +public class DSSProjectDO implements Serializable { + + private static final long serialVersionUID=1L; + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + private String name; + + /** + * Source of the dss_project + */ + private String source; + + private String description; + + private Long userId; + + private String username; + + private Long workspaceId; + + private Date createTime; + + private String createBy; + + private Date updateTime; + + private String updateBy; + + /** + * Organization ID + */ + private Long orgId; + + private Boolean visibility; + + /** + * Reserved word + */ + private Boolean isTransfer; + + private Long initialOrgId; + + /** + * If it is archived + */ + @TableField("isArchive") + private Boolean isArchive; + + private String pic; + + private Integer starNum; + + private String product; + + private Integer applicationArea; + + private String business; + + private Integer isPersonal; + + private String createByStr; + + private String updateByStr; + /** + * 开发流程,多个以英文逗号分隔,取得的值是dss_dictionary中的dic_key(parent_key=p_develop_process),首尾以英文逗号结束 + */ + private String devProcess; + + /** + * 编码模式,多个以英文逗号分隔,取得的值是dss_dictionary中的dic_key(parent_key=p_arrangement_mode或下面一级),首尾以英文逗号结束 + */ + private String orchestratorMode; + + private Integer visible; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Long getUserId() { + return userId; + } + + public void setUserId(Long userId) { + this.userId = userId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + public Long getOrgId() { + return orgId; + } + + public void setOrgId(Long orgId) { + this.orgId = orgId; + } + + public Boolean getVisibility() { + return visibility; + } + + public void setVisibility(Boolean visibility) { + this.visibility = visibility; + } + + public Boolean getTransfer() { + return isTransfer; + } + + public void setTransfer(Boolean transfer) { + isTransfer = transfer; + } + + public Long getInitialOrgId() { + return initialOrgId; + } + + public void setInitialOrgId(Long initialOrgId) { + this.initialOrgId = initialOrgId; + } + + public Boolean getArchive() { + return isArchive; + } + + public void setArchive(Boolean archive) { + isArchive = archive; + } + + public String getPic() { + return pic; + } + + public void setPic(String pic) { + this.pic = pic; + } + + public Integer getStarNum() { + return starNum; + } + + public void setStarNum(Integer starNum) { + this.starNum = starNum; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public Integer getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(Integer applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public Integer getIsPersonal() { + return isPersonal; + } + + public void setIsPersonal(Integer isPersonal) { + this.isPersonal = isPersonal; + } + + public String getCreateByStr() { + return createByStr; + } + + public void setCreateByStr(String createByStr) { + this.createByStr = createByStr; + } + + public String getUpdateByStr() { + return updateByStr; + } + + public void setUpdateByStr(String updateByStr) { + this.updateByStr = updateByStr; + } + + public String getDevProcess() { + return devProcess; + } + + public void setDevProcess(String devProcess) { + this.devProcess = devProcess; + } + + public String getOrchestratorMode() { + return orchestratorMode; + } + + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } + + public Integer getVisible() { + return visible; + } + + public void setVisible(Integer visible) { + this.visible = visible; + } + + @Override + public String toString() { + return "DSSProjectDO{" + + "id=" + id + + ", name='" + name + '\'' + + ", source='" + source + '\'' + + ", description='" + description + '\'' + + ", userId=" + userId + + ", username='" + username + '\'' + + ", workspaceId=" + workspaceId + + ", createTime=" + createTime + + ", createBy='" + createBy + '\'' + + ", updateTime=" + updateTime + + ", updateBy='" + updateBy + '\'' + + ", orgId=" + orgId + + ", visibility=" + visibility + + ", isTransfer=" + isTransfer + + ", initialOrgId=" + initialOrgId + + ", isArchive=" + isArchive + + ", pic='" + pic + '\'' + + ", starNum=" + starNum + + ", product='" + product + '\'' + + ", applicationArea=" + applicationArea + + ", business='" + business + '\'' + + ", isPersonal=" + isPersonal + + ", createByStr='" + createByStr + '\'' + + ", updateByStr='" + updateByStr + '\'' + + ", devProcess='" + devProcess + '\'' + + ", orchestratorMode='" + orchestratorMode + '\'' + + ", visible=" + visible + + '}'; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectUser.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectUser.java new file mode 100644 index 000000000..9494d501e --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectUser.java @@ -0,0 +1,106 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.io.Serializable; +import java.util.Date; + + +@TableName(value = "dss_project_user") +public class DSSProjectUser implements Serializable { + + private static final long serialVersionUID=1L; + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + private Long projectId; + + private String username; + + private Long workspaceId; + + /** + * 权限等级:1-查看,2-编辑,3-发布 + */ + private Integer priv; + + private Date lastUpdateTime; + + public DSSProjectUser() { + } + + + public DSSProjectUser(Long workspaceId, Long projectId, String username, int priv) { + this.workspaceId = workspaceId; + this.projectId = projectId; + this.username = username; + this.priv = priv; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Integer getPriv() { + return priv; + } + + public void setPriv(Integer priv) { + this.priv = priv; + } + + public Date getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(Date lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/DSSProjectPo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/DSSProjectPo.java new file mode 100644 index 000000000..f6d6513ec --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/DSSProjectPo.java @@ -0,0 +1,276 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.po; + +import java.util.Date; +import java.util.List; + + +public class DSSProjectPo { + + + private Long id; + private String name; + private String description; + private String source; + private Date createTime; + private String createBy; //兼容Visualis 后修改类型 + private Date updateTime; + private Long orgID; + private Boolean visibility; + private Boolean isTransfer; + private Boolean isArchive; + private Long initialOrgID; + private String pic; + private Long starNum; + private String product; + private Integer applicationArea; + private String business; + private Integer workspaceId; + private String projectGroup; + private String workspaceName; + private String username; + private Integer isPersonal; + + private Long userID; + private String createByStr; + private String updateByStr; + + private List accessUsers; + private List editUsers; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public Long getOrgID() { + return orgID; + } + + public void setOrgID(Long orgID) { + this.orgID = orgID; + } + + public Boolean getVisibility() { + return visibility; + } + + public void setVisibility(Boolean visibility) { + this.visibility = visibility; + } + + public Boolean getTransfer() { + return isTransfer; + } + + public void setTransfer(Boolean transfer) { + isTransfer = transfer; + } + + public Boolean getArchive() { + return isArchive; + } + + public void setArchive(Boolean archive) { + isArchive = archive; + } + + public Long getInitialOrgID() { + return initialOrgID; + } + + public void setInitialOrgID(Long initialOrgID) { + this.initialOrgID = initialOrgID; + } + + public String getPic() { + return pic; + } + + public void setPic(String pic) { + this.pic = pic; + } + + public Long getStarNum() { + return starNum; + } + + public void setStarNum(Long starNum) { + this.starNum = starNum; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public Integer getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(Integer applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public String getProjectGroup() { + return projectGroup; + } + + public void setProjectGroup(String projectGroup) { + this.projectGroup = projectGroup; + } + + public List getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Integer getIsPersonal() { + return isPersonal; + } + + public void setIsPersonal(Integer isPersonal) { + this.isPersonal = isPersonal; + } + + public Long getUserID() { + return userID; + } + + public void setUserID(Long userID) { + this.userID = userID; + } + + public String getCreateByStr() { + return createByStr; + } + + public void setCreateByStr(String createByStr) { + this.createByStr = createByStr; + } + + public String getUpdateByStr() { + return updateByStr; + } + + public void setUpdateByStr(String updateByStr) { + this.updateByStr = updateByStr; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/ProjectRelationPo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/ProjectRelationPo.java new file mode 100644 index 000000000..d7648d0f4 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/ProjectRelationPo.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.po; + + +public class ProjectRelationPo { + + private Long dssProjectId; + + private Long appInstanceId; + + private Long appInstanceProjectId; + + + public ProjectRelationPo(Long dssProjectId, Long appInstanceId, Long appInstanceProjectId) { + this.dssProjectId = dssProjectId; + this.appInstanceId = appInstanceId; + this.appInstanceProjectId = appInstanceProjectId; + } + + public Long getDssProjectId() { + return dssProjectId; + } + + public void setDssProjectId(Long dssProjectId) { + this.dssProjectId = dssProjectId; + } + + public Long getAppInstanceId() { + return appInstanceId; + } + + public void setAppInstanceId(Long appInstanceId) { + this.appInstanceId = appInstanceId; + } + + public Long getAppInstanceProjectId() { + return appInstanceProjectId; + } + + public void setAppInstanceProjectId(Long appInstanceProjectId) { + this.appInstanceProjectId = appInstanceProjectId; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectCreateRequest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectCreateRequest.java new file mode 100644 index 000000000..4d9710881 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectCreateRequest.java @@ -0,0 +1,160 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + + +@XmlRootElement +public class ProjectCreateRequest { + + @NotNull(message = "工程名称不能为空") + private String name; + + @NotNull(message = "应用领域不能为空") + private Integer applicationArea; + + private String business; + + //产品 + private String product; + + private List releaseUsers; + + private List editUsers; + + private List accessUsers; + + @NotNull(message = "工程描述不能为空") + private String description; + + /** + * 工作空间名,因为是全局唯一的 + */ + @NotNull(message = "workspaceId不能为空") + private Long workspaceId; + + private String workspaceName; + /** + * 开发流程 list + */ + private List devProcessList; + + /** + * 编码模式 list + */ + private List orchestratorModeList; + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(Integer applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public List getReleaseUsers() { + return releaseUsers; + } + + public void setReleaseUsers(List releaseUsers) { + this.releaseUsers = releaseUsers; + } + + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + public List getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public List getDevProcessList() { + return devProcessList; + } + + public void setDevProcessList(List devProcessList) { + this.devProcessList = devProcessList; + } + + public List getOrchestratorModeList() { + return orchestratorModeList; + } + + public void setOrchestratorModeList(List orchestratorModeList) { + this.orchestratorModeList = orchestratorModeList; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectDeleteRequest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectDeleteRequest.java new file mode 100644 index 000000000..031521267 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectDeleteRequest.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; + + +@XmlRootElement +public class ProjectDeleteRequest { + + @NotNull(message = "工程id不能为空") + private Long id; + + @NotNull(message = "确认不能为空") + private boolean sure; + + @NotNull(message = "同步删除其它系统工程不能为空") + private boolean ifDelOtherSys; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public boolean isSure() { + return sure; + } + + public void setSure(boolean sure) { + this.sure = sure; + } + + public boolean isIfDelOtherSys() { + return ifDelOtherSys; + } + + public void setIfDelOtherSys(boolean ifDelOtherSys) { + this.ifDelOtherSys = ifDelOtherSys; + } + + @Override + public String toString() { + return "ProjectDeleteRequest{" + + "id=" + id + + ", sure=" + sure + + ", ifDelOtherSys=" + ifDelOtherSys + + '}'; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectModifyRequest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectModifyRequest.java new file mode 100644 index 000000000..78b28ab61 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectModifyRequest.java @@ -0,0 +1,194 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + + +@XmlRootElement +public class ProjectModifyRequest { + + + @NotNull(message = "工程名称不能为空") + private String name; + + @NotNull(message = "工程id不能为空") + private Long id; + + @NotNull(message = "应用领域不能为空") + private String applicationArea; + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + private String workspaceName; + + //业务 + private String business; + + //编辑权限用户 + private List editUsers; + + //查看权限用户 + private List accessUsers; + + //发布权限用户 + private List releaseUsers; + + @NotNull(message = "工程描述不能为空") + private String description; + + //产品 + private String product; + + /** + * 工作空间名,因为是全局唯一的 + */ + @NotNull(message = "workspaceId不能为空") + private Long workspaceId; + + /* + 开发流程 code list + */ + private List devProcessList; + + /** + * 编排模式 code list + */ + private List orchestratorModeList; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(String applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + public List getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public List getReleaseUsers() { + return releaseUsers; + } + + public void setReleaseUsers(List releaseUsers) { + this.releaseUsers = releaseUsers; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public List getDevProcessList() { + return devProcessList; + } + + public void setDevProcessList(List devProcessList) { + this.devProcessList = devProcessList; + } + + public List getOrchestratorModeList() { + return orchestratorModeList; + } + + public void setOrchestratorModeList(List orchestratorModeList) { + this.orchestratorModeList = orchestratorModeList; + } + + @Override + public String toString() { + return "ProjectModifyRequest{" + + "name='" + name + '\'' + + ", id=" + id + + ", applicationArea='" + applicationArea + '\'' + + ", business='" + business + '\'' + + ", editUsers=" + editUsers + + ", accessUsers=" + accessUsers + + ", releaseUsers=" + releaseUsers + + ", description='" + description + '\'' + + ", product='" + product + '\'' + + ", workspaceId=" + workspaceId + + ", devProcessList=" + devProcessList + + ", orchestratorModeList=" + orchestratorModeList + + '}'; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectQueryRequest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectQueryRequest.java new file mode 100644 index 000000000..b0147415a --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectQueryRequest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; + + +@XmlRootElement +public class ProjectQueryRequest implements Serializable { + + private static final long serialVersionUID=1L; + + private Long id; + + @NotNull(message = "workspaceId不能为空") + private Long workspaceId; + + private String username; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ProjectResponse.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ProjectResponse.java new file mode 100644 index 000000000..e049ffde4 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ProjectResponse.java @@ -0,0 +1,233 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.response; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + + +public class ProjectResponse implements Serializable { + private static final long serialVersionUID=1L; + //工程id + private Long id; + //应用领域 + private Integer applicationArea; + //业务 + private String business; + //创建人 + private String createBy; + //工程描述 + private String description; + //工程名称 + private String name; + // + private String source; + //产品 + private String product; + private Boolean isArchive; + //工程创建时间 + private Date createTime; + //工程修改时间 + private Date updateTime; + + public Boolean getEditable() { + return editable; + } + + public void setEditable(Boolean editable) { + this.editable = editable; + } + + private Boolean editable; + + /** + * 发布用户 list + */ + private List releaseUsers; + + /** + * 编辑用户 list + */ + private List editUsers; + + /** + * 查看用户 list + */ + private List accessUsers; + + /** + * 开发流程 list + */ + private List devProcessList; + + /** + * 编码模式 list + */ + private List orchestratorModeList; + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Integer getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(Integer applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public Boolean getArchive() { + return isArchive; + } + + public void setArchive(Boolean archive) { + isArchive = archive; + } + + public List getReleaseUsers() { + return releaseUsers; + } + + public void setReleaseUsers(List releaseUsers) { + this.releaseUsers = releaseUsers; + } + + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + public List getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public List getDevProcessList() { + return devProcessList; + } + + public void setDevProcessList(List devProcessList) { + this.devProcessList = devProcessList; + } + + public List getOrchestratorModeList() { + return orchestratorModeList; + } + + public void setOrchestratorModeList(List orchestratorModeList) { + this.orchestratorModeList = orchestratorModeList; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "ResponseProjectVo{" + + "id=" + id + + ", applicationArea=" + applicationArea + + ", business='" + business + '\'' + + ", createBy='" + createBy + '\'' + + ", description='" + description + '\'' + + ", name='" + name + '\'' + + ", source='" + source + '\'' + + ", product='" + product + '\'' + + ", isArchive=" + isArchive + + ", createTime=" + createTime + + ", updateTime=" + updateTime + + ", releaseUsers=" + releaseUsers + + ", editUsers=" + editUsers + + ", accessUsers=" + accessUsers + + ", devProcessList=" + devProcessList + + ", orchestratorModeList=" + orchestratorModeList + + ", editable=" + editable + + '}'; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/DSSProjectDetailVo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/DSSProjectDetailVo.java new file mode 100644 index 000000000..cf5a85033 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/DSSProjectDetailVo.java @@ -0,0 +1,109 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.vo; + +import java.util.List; + + + +public class DSSProjectDetailVo extends DSSProjectVo { + + /** + * 发布用户 list + */ + private List releaseUsers; + + /** + * 编辑用户 list + */ + private List editUsers; + + /** + * 查看用户 list + */ + private List accessUsers; + + /** + * 开发的模式,主要是工作流和应用工具 + */ + private String projectMode; + + /** + * 应用领域,不为空 + */ + private String applicationArea; + + + /** + * 使用业务 + */ + private String business; + + + private String description; + + + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + public List getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public String getProjectMode() { + return projectMode; + } + + public void setProjectMode(String projectMode) { + this.projectMode = projectMode; + } + + public String getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(String applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + @Override + public String getDescription() { + return description; + } + + @Override + public void setDescription(String description) { + this.description = description; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/DSSProjectVo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/DSSProjectVo.java new file mode 100644 index 000000000..a7206d452 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/DSSProjectVo.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.vo; + +import java.util.List; + + +public class DSSProjectVo { + + private Long id; + + private String name; + + private String description; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/ProcessNode.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/ProcessNode.java new file mode 100644 index 000000000..094c32a85 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/ProcessNode.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.vo; + + +public interface ProcessNode { + + Integer getIndex(); + + String getName(); + + String getChName(); + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/ProjectInfoVo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/ProjectInfoVo.java new file mode 100644 index 000000000..abe420c48 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/ProjectInfoVo.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.vo; + +import java.io.Serializable; + + +public class ProjectInfoVo implements Serializable { + + private static final long serialVersionUID=1L; + private Long id; + private Long workspaceId; + private String projectName; + private String workspaceName; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/QueryProjectVo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/QueryProjectVo.java new file mode 100644 index 000000000..687b6d361 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/QueryProjectVo.java @@ -0,0 +1,165 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.vo; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + + +public class QueryProjectVo implements Serializable { + + private static final long serialVersionUID=1L; + private Long id; + private Integer applicationArea; + private String business; + private String createBy; + private String description; + private String name; + private String source; + private String product; + private Boolean isArchive; + private Date createTime; + private Date updateTime; + + private String devProcess; + private String orchestratorMode; + private String pusername; + + private Integer visible; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Integer getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(Integer applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public Boolean getArchive() { + return isArchive; + } + + public void setArchive(Boolean archive) { + isArchive = archive; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getDevProcess() { + return devProcess; + } + + public void setDevProcess(String devProcess) { + this.devProcess = devProcess; + } + + public String getOrchestratorMode() { + return orchestratorMode; + } + + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } + + public String getPusername() { + return pusername; + } + + public void setPusername(String pusername) { + this.pusername = pusername; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public Integer getVisible() { + return visible; + } + + public void setVisible(Integer visible) { + this.visible = visible; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/DSSProjectErrorException.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/DSSProjectErrorException.java new file mode 100644 index 000000000..bd2f1cddd --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/DSSProjectErrorException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class DSSProjectErrorException extends ErrorException { + + public DSSProjectErrorException(int errorCode, String errorDesc){ + super(errorCode, errorDesc); + } + + public DSSProjectErrorException(int errorCode, String errorDesc, Exception e) { + super(errorCode, errorDesc); + initCause(e); + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/DSSProjectWarnException.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/DSSProjectWarnException.java new file mode 100644 index 000000000..5ec10360a --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/DSSProjectWarnException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.exception; + +import org.apache.linkis.common.exception.WarnException; + + +public class DSSProjectWarnException extends WarnException { + public DSSProjectWarnException(int errorCode, String errorDesc){ + super(errorCode, errorDesc); + } + + + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/LambdaWarnException.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/LambdaWarnException.java new file mode 100644 index 000000000..f14debcd0 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/exception/LambdaWarnException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.exception; + + +public class LambdaWarnException extends DSSProjectWarnException { + + + public LambdaWarnException(int errorCode, String errorDesc) { + super(errorCode, errorDesc); + } + + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/restful/DSSFrameworkProjectRestfulApi.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/restful/DSSFrameworkProjectRestfulApi.java new file mode 100644 index 000000000..866b800ad --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/restful/DSSFrameworkProjectRestfulApi.java @@ -0,0 +1,197 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.restful; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectDeleteRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; +import com.webank.wedatasphere.dss.framework.project.entity.response.ProjectResponse; +import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; +import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; +import com.webank.wedatasphere.dss.framework.project.service.DSSFrameworkProjectService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; +import com.webank.wedatasphere.dss.framework.project.utils.ApplicationArea; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import javax.validation.Valid; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RequestMapping(path = "/dss/framework/project", produces = {"application/json"}) +@RestController +public class DSSFrameworkProjectRestfulApi { + private static final Logger LOGGER = LoggerFactory.getLogger(DSSFrameworkProjectRestfulApi.class); + @Autowired + DSSFrameworkProjectService dssFrameworkProjectService; + @Autowired + private DSSProjectService projectService; + @Autowired + private DSSProjectService dssProjectService; + + /** + * 获取所有工程或者单个工程 + * + * @param request + * @param projectRequest + * @return + */ + @RequestMapping(path = "getAllProjects", method = RequestMethod.POST) + public Message getAllProjects(HttpServletRequest request, @RequestBody ProjectQueryRequest projectRequest) { + String username = SecurityFilter.getLoginUsername(request); + projectRequest.setUsername(username); + List dssProjectVos = projectService.getListByParam(projectRequest); + Message message = Message.ok("获取工作空间的工程成功").data("projects", dssProjectVos); + return message; + } + + /** + * 新建工程,通过和各个AppConn进行交互,将需要满足工程规范的所有的appconn进行创建工程 + */ + @RequestMapping(path = "createProject", method = RequestMethod.POST) + public Message createProject(HttpServletRequest request, @RequestBody ProjectCreateRequest projectCreateRequest) throws Exception { + String username = SecurityFilter.getLoginUsername(request); + Workspace workspace = SSOHelper.getWorkspace(request); + //將创建人默认为发布权限和編輯权限 + if (!projectCreateRequest.getEditUsers().contains(username)) { + projectCreateRequest.getEditUsers().add(username); + } + List releaseUsers = projectCreateRequest.getReleaseUsers(); + if (releaseUsers == null) { + releaseUsers = new ArrayList<>(); + projectCreateRequest.setReleaseUsers(releaseUsers); + } + if (!releaseUsers.contains(username)) { + releaseUsers.add(username); + } + DSSProjectVo dssProjectVo = dssFrameworkProjectService.createProject(projectCreateRequest, username, workspace); + if (dssProjectVo != null) { + return Message.ok("创建工程成功").data("project", dssProjectVo); + } else { + return Message.error("创建工程失败"); + } + } + + /** + * 编辑工程 + * + * @param request + * @param projectModifyRequest + * @return + */ + @RequestMapping(path = "modifyProject", method = RequestMethod.POST) + public Message modifyProject(HttpServletRequest request, @RequestBody ProjectModifyRequest projectModifyRequest) throws Exception { + String username = SecurityFilter.getLoginUsername(request); + Workspace workspace = SSOHelper.getWorkspace(request); + DSSProjectDO dbProject = dssProjectService.getProjectById(projectModifyRequest.getId()); + //工程不存在 + if (dbProject == null) { + LOGGER.error("{} project id is null, can not modify", projectModifyRequest.getName()); + DSSExceptionUtils.dealErrorException(60021, + String.format("%s project id is null, can not modify", projectModifyRequest.getName()), DSSProjectErrorException.class); + } + String createUsername = dbProject.getUsername(); + //將创建人默认为发布权限和編輯权限 + if (!projectModifyRequest.getEditUsers().contains(createUsername)) { + projectModifyRequest.getEditUsers().add(createUsername); + } + List releaseUsers = projectModifyRequest.getReleaseUsers(); + if (releaseUsers == null) { + releaseUsers = new ArrayList<>(); + projectModifyRequest.setReleaseUsers(releaseUsers); + } + if (!releaseUsers.contains(createUsername)) { + releaseUsers.add(createUsername); + } + dssFrameworkProjectService.modifyProject(projectModifyRequest, dbProject, username, workspace); + return Message.ok("修改工程成功"); + } + + /** + * 删除工程 + * + * @param request + * @param projectDeleteRequest + * @return + */ + @RequestMapping(path = "deleteProject", method = RequestMethod.POST) + public Message deleteProject(HttpServletRequest request, @RequestBody ProjectDeleteRequest projectDeleteRequest) throws Exception { + String username = SecurityFilter.getLoginUsername(request); + Workspace workspace = SSOHelper.getWorkspace(request); + // 检查是否具有删除项目权限 + projectService.isDeleteProjectAuth(projectDeleteRequest.getId(), username); + projectService.deleteProject(username, projectDeleteRequest, workspace); + return Message.ok("删除工程成功"); + } + + @RequestMapping(path = "listApplicationAreas", method = RequestMethod.GET) + public Message listApplicationAreas(HttpServletRequest req) { + String header = req.getHeader("Content-language").trim(); + ApplicationArea[] applicationAreas = ApplicationArea.values(); + List areas = new ArrayList<>(); + Arrays.stream(applicationAreas).forEach(item -> { + if ("zh-CN".equals(header)) { + areas.add(item.getName()); + } else { + areas.add(item.getEnName()); + } + }); + return Message.ok().data("applicationAreas", areas); + } + + @RequestMapping(path = "getProjectAbilities", method = RequestMethod.GET) + public Message getProjectAbilities(HttpServletRequest request) { + //为了获取到此环境的能力,导入 导出 发布等 + String username = SecurityFilter.getLoginUsername(request); + try { + List projectAbilities = projectService.getProjectAbilities(username); + return Message.ok("获取工程能力成功").data("projectAbilities", projectAbilities); + } catch (final Throwable t) { + LOGGER.error("failed to get project ability for user {}", username, t); + return Message.error("获取工程能力失败"); + } + } + + + /** + * 获取已删除的所有工程 + */ + @RequestMapping(path = "getDeletedProjects", method = RequestMethod.POST) + public Message getDeletedProjects(HttpServletRequest request, @Valid @RequestBody ProjectQueryRequest projectRequest) { + String username = SecurityFilter.getLoginUsername(request); + projectRequest.setUsername(username); + List dssProjectVos = projectService.getDeletedProjects(projectRequest); + Message message = Message.ok("获取工作空间已删除的工程成功").data("projects", dssProjectVos); + return message; + } + + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSFrameworkProjectService.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSFrameworkProjectService.java new file mode 100644 index 000000000..a55fe7619 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSFrameworkProjectService.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service; + +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectDetailVo; +import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +public interface DSSFrameworkProjectService { + + DSSProjectDetailVo getProjectSettings(Long projectId); + + /** + * 1. 首先要去所有满足工程结构的规范的去建工程,首先必须要满足建工程的 AppConn 进行进工程 + * 2. 自己创建工程 + * 3. 如果第三方系统创建失败,最多重试一次 + * 4. 如果本身失败了,则进行回滚 + * @param projectCreateRequest + * @return + */ + DSSProjectVo createProject(ProjectCreateRequest projectCreateRequest, String username, Workspace workspace) throws Exception; + + void modifyProject(ProjectModifyRequest projectModifyRequest, DSSProjectDO dbProject, String username, Workspace workspace) throws Exception; +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectService.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectService.java new file mode 100644 index 000000000..477a08cba --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectService.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectDeleteRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; +import com.webank.wedatasphere.dss.framework.project.entity.response.ProjectResponse; +import com.webank.wedatasphere.dss.framework.project.entity.vo.ProjectInfoVo; +import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestProjectImportOrchestrator; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +import java.util.List; +import java.util.Map; + +public interface DSSProjectService extends IService { + + + DSSProjectDO createProject(String username, ProjectCreateRequest projectCreateRequest); + + + void modifyProject(String username, ProjectModifyRequest modifyRequest) throws DSSProjectErrorException; + + /** + * 旧工程导入到新环境的,修改新环境工程相关字段 + * @param updateProject 旧工程(91) + * @param dbProject 数据库工程(246) + * @throws Exception + */ + void modifyOldProject(DSSProjectDO updateProject, DSSProjectDO dbProject); + + DSSProjectDO getProjectByName(String name); + + + DSSProjectDO getProjectById(Long id); + + + List getListByParam(ProjectQueryRequest projectRequest); + + + ProjectInfoVo getProjectInfoById(Long id); + + void saveProjectRelation(DSSProjectDO project, Map projectMap); + + Long getAppConnProjectId(Long dssProjectId, String appConnName, List dssLabels) throws Exception; + + Long getAppConnProjectId(Long appInstanceId, Long dssProjectId); + + void deleteProject(String username, ProjectDeleteRequest projectDeleteRequest, Workspace workspace) throws Exception; + + List getProjectAbilities(String username); + + + boolean isDeleteProjectAuth(Long projectId, String username) throws DSSProjectErrorException ; + + /** + * 查询已删除工程list + * @param projectRequest + * @return + */ + List getDeletedProjects(ProjectQueryRequest projectRequest); + + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectUserService.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectUserService.java new file mode 100644 index 000000000..b012ff17e --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectUserService.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service; + +import java.util.List; + +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectUser; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +public interface DSSProjectUserService { + + /** + * 是否有修改工程权限 + * + * @param projectId + * the project id + * @param username + * the username + * @return boolean + * @throws DSSProjectErrorException + * the dss project error exception + */ + boolean isEditProjectAuth(Long projectId,String username) throws DSSProjectErrorException; + + /** + * 保存工程与用户关系 + * + * @param projectID + * the project id + * @param username + * the username + * @param dssProjectCreateRequest + * the dss project create request + * @throws Exception + * the exception + */ + void saveProjectUser(Long projectID, String username, ProjectCreateRequest dssProjectCreateRequest, Workspace workspace)throws Exception; + + /** + * 修改工程与用户关系 + * + * @param dbProject + * the db project + * @param projectModifyRequest + * the project modify request + * @param loginuser + * the loginuser + * @throws Exception + * the exception + */ + void modifyProjectUser(DSSProjectDO dbProject, ProjectModifyRequest projectModifyRequest, String loginuser,Workspace workspace)throws Exception; + + + List getListByParam(Long workspaceId, String username); + + + boolean isAdminByUsername(Long workspaceId,String username); + + List getProjectUserPriv(Long projectId, String username); + + List getProjectPriv(Long projectId); + + boolean isWorkspaceUser(Long workspaceId,String username); + + /** + * 获取某个项目下指定权限的用户名合. + * + * @param projectId + * the project id + * @param privEnum + * the priv enum + * @return the list + */ + public List listByPriv(Long projectId, ProjectUserPrivEnum privEnum); +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSFrameworkProjectServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSFrameworkProjectServiceImpl.java new file mode 100644 index 000000000..9b496ea14 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSFrameworkProjectServiceImpl.java @@ -0,0 +1,268 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyStructureAppConn; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.framework.project.contant.ProjectServerResponse; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectUser; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectDetailVo; +import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; +import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; +import com.webank.wedatasphere.dss.framework.project.service.DSSFrameworkProjectService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectUserService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.structure.project.*; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectPrivilege; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.utils.StructureOperationUtils; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.protocol.util.ImmutablePair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; + +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.framework.project.utils.ProjectOperationUtils.tryProjectOperation; + +public class DSSFrameworkProjectServiceImpl implements DSSFrameworkProjectService { + private static final Logger LOGGER = LoggerFactory.getLogger(DSSFrameworkProjectServiceImpl.class); + public static final int MAX_PROJECT_NAME_SIZE = 150; + public static final int MAX_PROJECT_DESC_SIZE = 2048; + @Autowired + private DSSProjectService dssProjectService; + @Autowired + private DSSProjectUserService projectUserService; + + + private static final boolean STRICT_PROJECT_CREATE_MODE = CommonVars.apply("wds.dss.project.strict.mode", false).getValue(); + + @Override + public DSSProjectDetailVo getProjectSettings(Long projectId) { + return null; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DSSProjectVo createProject(ProjectCreateRequest projectCreateRequest, String username, Workspace workspace) throws Exception { + //1.新建DSS工程,这样才能进行回滚,如果后面去DSS工程,可能会由于DSS工程建立失败了,但是仍然无法去回滚第三方系统的工程 + //2.开始创建appconn的相关的工程,如果失败了,抛异常,然后进行数据库进行回滚 + boolean isWorkspaceUser = projectUserService.isWorkspaceUser(projectCreateRequest.getWorkspaceId(), username); + //非管理员 + if (!isWorkspaceUser) { + DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_USER_NOT_IN_WORKSPACE.getCode(), ProjectServerResponse.PROJECT_USER_NOT_IN_WORKSPACE.getMsg(), DSSProjectErrorException.class); + } + //增加名称长度限制 + if(projectCreateRequest.getName().length()> MAX_PROJECT_NAME_SIZE || projectCreateRequest.getDescription().length() > MAX_PROJECT_DESC_SIZE){ + DSSExceptionUtils.dealErrorException(60021,"project name or desc is too long for project name length is " + projectCreateRequest.getName().length() + + ", project desc length is " + projectCreateRequest.getDescription().length(), DSSProjectErrorException.class); + } + + //判断工程是否存在相同的名称 + DSSProjectDO dbProject = dssProjectService.getProjectByName(projectCreateRequest.getName()); + if (dbProject != null) { + DSSExceptionUtils.dealErrorException(60022, String.format("project name %s has already been exists.", projectCreateRequest.getName()), DSSProjectErrorException.class); + } + + List appConnNameList = new ArrayList<>(1); + //判断已有组件是否已经存在相同的工程名称 + try { + isExistSameProjectName(projectCreateRequest, workspace, appConnNameList, username); + } catch (Exception e) { + throw new DSSProjectErrorException(71000, "向第三方应用发起检查工程名是否重复失败,原因:" + ExceptionUtils.getRootCauseMessage(e), e); + } + if (!appConnNameList.isEmpty()) { + throw new DSSProjectErrorException(71000, String.join(", ", appConnNameList) + " 已存在相同项目名称,请重新命名!"); + } + + Map projectMap = createAppConnProject(projectCreateRequest, workspace, username); + //3.保存dss_project + DSSProjectDO project = dssProjectService.createProject(username, projectCreateRequest); + //4.保存dss_project_user 工程与用户关系 + projectUserService.saveProjectUser(project.getId(), username, projectCreateRequest, workspace); + //5.保存dss工程与其他工程的对应关系,应该都是以id来作为标识 + if (projectMap.size() > 0) { + dssProjectService.saveProjectRelation(project, projectMap); + } + DSSProjectVo dssProjectVo = new DSSProjectVo(); + dssProjectVo.setDescription(project.getDescription()); + dssProjectVo.setId(project.getId()); + dssProjectVo.setName(project.getName()); + return dssProjectVo; + } + + + @Override + public void modifyProject(ProjectModifyRequest projectModifyRequest, DSSProjectDO dbProject, String username, Workspace workspace) throws Exception { + //如果不是工程的创建人,则校验是否管理员 + if (!username.equalsIgnoreCase(dbProject.getCreateBy())) { + boolean isAdmin = projectUserService.isAdminByUsername(projectModifyRequest.getWorkspaceId(), username); + //非管理员 + if (!isAdmin) { + DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_IS_NOT_ADMIN.getCode(), ProjectServerResponse.PROJECT_IS_NOT_ADMIN.getMsg(), DSSProjectErrorException.class); + } + } + //不允许修改工程名称 + if (!dbProject.getName().equalsIgnoreCase(projectModifyRequest.getName())) { + DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_NOT_EDIT_NAME.getCode(), ProjectServerResponse.PROJECT_NOT_EDIT_NAME.getMsg(), DSSProjectErrorException.class); + } + //调用第三方的工程修改接口 + dbProject.setUsername(username); + modifyThirdProject(projectModifyRequest, dbProject, workspace); + + //1.统一修改各个接入的第三方的系统的工程状态信息 + //2.修改dss_project_user 工程与用户关系 + projectUserService.modifyProjectUser(dbProject, projectModifyRequest, username, workspace); + + //3.修改dss_project DSS基本工程信息 + dssProjectService.modifyProject(username, projectModifyRequest); + } + + /** + * 统一修改各个接入的第三方的系统的工程状态信息,修改 dss_project 调用。 + */ + private void modifyThirdProject(ProjectModifyRequest projectModifyRequest, + DSSProjectDO dbProject, Workspace workspace){ + DSSProject dssProject = new DSSProject(); + BeanUtils.copyProperties(dbProject, dssProject); + DSSProjectPrivilege privilege = DSSProjectPrivilege.newBuilder().setAccessUsers(projectModifyRequest.getAccessUsers()) + .setEditUsers(projectModifyRequest.getEditUsers()) + .setReleaseUsers(projectModifyRequest.getReleaseUsers()).build(); + List projectUsers = projectUserService.getProjectPriv(projectModifyRequest.getId()); + DSSProjectPrivilege addedPrivilege; + DSSProjectPrivilege removedPrivilege; + if(CollectionUtils.isEmpty(projectUsers)) { + addedPrivilege = privilege; + removedPrivilege = DSSProjectPrivilege.EMPTY; + } else { + BiFunction, List, List> getDifference = (privAndIsAdded, users) -> { + List privUsers = projectUsers.stream().filter(user -> user.getPriv().equals(privAndIsAdded.getKey())).map(DSSProjectUser::getUsername).collect(Collectors.toList()); + if(privAndIsAdded.getValue()) { + return (List) CollectionUtils.subtract(users, privUsers); + } else { + return (List) CollectionUtils.subtract(privUsers, users); + } + }; + Function getPrivilege = isAdded -> DSSProjectPrivilege.newBuilder().setAccessUsers(getDifference.apply(new ImmutablePair<>(1, isAdded), projectModifyRequest.getAccessUsers())) + .setEditUsers(getDifference.apply(new ImmutablePair<>(2, isAdded), projectModifyRequest.getEditUsers())) + .setReleaseUsers(getDifference.apply(new ImmutablePair<>(3, isAdded), projectModifyRequest.getReleaseUsers())).build(); + addedPrivilege = getPrivilege.apply(true); + removedPrivilege = getPrivilege.apply(false); + } + Map appInstanceToRefProjectId = new HashMap<>(10); + tryProjectOperation((appConn, appInstance) -> { + Long refProjectId = dssProjectService.getAppConnProjectId(appInstance.getId(), dbProject.getId()); + if(refProjectId == null) { + LOGGER.warn("update project {} for third-party AppConn {} is ignored, appInstance is {}. Caused by: the refProjectId is null.", + projectModifyRequest.getName(), appConn.getAppDesc().getAppName(), appInstance.getBaseUrl()); + return false; + } else { + appInstanceToRefProjectId.put(appInstance, refProjectId); + return true; + } + }, workspace, projectService -> projectService.getProjectUpdateOperation(), + dssProjectContentRequestRef -> dssProjectContentRequestRef.setDSSProject(dssProject).setDSSProjectPrivilege(privilege).setUserName(dbProject.getUpdateBy()).setWorkspace(workspace), + (appInstance, refProjectContentRequestRef) -> refProjectContentRequestRef.setRefProjectId(appInstanceToRefProjectId.get(appInstance)), + (structureOperation, structureRequestRef) -> { + ProjectUpdateRequestRef projectUpdateRequestRef = (ProjectUpdateRequestRef) structureRequestRef; + projectUpdateRequestRef.setAddedDSSProjectPrivilege(addedPrivilege).setRemovedDSSProjectPrivilege(removedPrivilege); + return ((ProjectUpdateOperation) structureOperation).updateProject(projectUpdateRequestRef); + }, null, "update refProject " + projectModifyRequest.getName()); + } + + + + private void isExistSameProjectName(ProjectCreateRequest dssProjectCreateRequest, + Workspace workspace, + List appConnNameList, + String username) throws ExternalOperationFailedException { + LOGGER.info("begin to check whether the project name {} is already exists in third-party AppConn...", dssProjectCreateRequest.getName()); + tryProjectOperation(null, workspace, ProjectService::getProjectSearchOperation, + null, + (appInstance, refProjectContentRequestRef) -> refProjectContentRequestRef.setProjectName(dssProjectCreateRequest.getName()).setUserName(username), + (structureOperation, structureRequestRef) -> ((ProjectSearchOperation) structureOperation).searchProject((RefProjectContentRequestRef) structureRequestRef), + (pair, responseRef) -> { + ProjectService projectService = ((OnlyStructureAppConn) pair.getLeft()).getOrCreateStructureStandard().getProjectService(pair.getRight()); + if(responseRef.getRefProjectId() != null && responseRef.getRefProjectId() > 0 && projectService.isProjectNameUnique()) { + appConnNameList.add(pair.getLeft().getAppDesc().getAppName()); + } + }, "check project name " + dssProjectCreateRequest.getName() + " whether third-party refProject is exists"); + } + + // 1.新建DSS工程,这样才能进行回滚,如果后面去DSS工程,可能会由于DSS工程建立失败了,但是仍然无法去回滚第三方系统的工程 新增dss_project调用 + private Map createAppConnProject(ProjectCreateRequest dssProjectCreateRequest, Workspace workspace, String username) { + final Map projectMap = new HashMap<>(10); + final Map> appConnListMap = new HashMap<>(10); + DSSProject dssProject = new DSSProject(); + BeanUtils.copyProperties(dssProjectCreateRequest, dssProject); + dssProject.setCreateBy(username); + dssProject.setCreateTime(new Date()); + dssProject.setUsername(username); + DSSProjectPrivilege privilege = DSSProjectPrivilege.newBuilder().setAccessUsers(dssProjectCreateRequest.getAccessUsers()) + .setEditUsers(dssProjectCreateRequest.getEditUsers()) + .setReleaseUsers(dssProjectCreateRequest.getReleaseUsers()).build(); + try { + tryProjectOperation(null, workspace, ProjectService::getProjectCreationOperation, + dssProjectContentRequestRef -> dssProjectContentRequestRef.setDSSProject(dssProject) + .setDSSProjectPrivilege(privilege).setUserName(username), null, + (structureOperation, structureRequestRef) -> ((ProjectCreationOperation) structureOperation).createProject((DSSProjectContentRequestRef) structureRequestRef), + (pair, projectResponseRef) -> { + projectMap.put(pair.right, projectResponseRef.getRefProjectId()); + if (!appConnListMap.containsKey(pair.left)) { + appConnListMap.put(pair.left, new ArrayList<>()); + } + appConnListMap.get(pair.left).add(pair.right); + }, "create refProject " + dssProjectCreateRequest.getName()); + } catch (RuntimeException e) { + LOGGER.error("create appconn project failed:", e); + if(!STRICT_PROJECT_CREATE_MODE) { + throw e; + } + // 如果创建失败并且是严格创建模式 + // 如果一个AppInstance实例是失败的,那么我们将所有已经建的工程给撤销掉 + appConnListMap.forEach((key, value) -> value.forEach(appInstance -> { + StructureOperationUtils.tryProjectOperation(() -> ((OnlyStructureAppConn) key).getOrCreateStructureStandard().getProjectService(appInstance), + ProjectService::getProjectDeletionOperation, null, + refProjectContentRequestRef -> refProjectContentRequestRef.setRefProjectId(projectMap.get(appInstance)) + .setProjectName(dssProjectCreateRequest.getName()).setWorkspace(workspace).setUserName(username), + (structureOperation, structureRequestRef) -> ((ProjectDeletionOperation) structureOperation).deleteProject((RefProjectContentRequestRef) structureRequestRef), + "delete refProject " + dssProjectCreateRequest.getName()); + })); + throw e; + } + return projectMap; + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java new file mode 100644 index 000000000..09c895930 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java @@ -0,0 +1,364 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.framework.project.conf.ProjectConf; +import com.webank.wedatasphere.dss.framework.project.contant.ProjectServerResponse; +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.framework.project.dao.DSSProjectMapper; +import com.webank.wedatasphere.dss.framework.project.dao.DSSProjectUserMapper; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.po.ProjectRelationPo; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectDeleteRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; +import com.webank.wedatasphere.dss.framework.project.entity.response.ProjectResponse; +import com.webank.wedatasphere.dss.framework.project.entity.vo.ProjectInfoVo; +import com.webank.wedatasphere.dss.framework.project.entity.vo.QueryProjectVo; +import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectUserService; +import com.webank.wedatasphere.dss.framework.project.utils.ProjectStringUtils; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.*; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.framework.project.utils.ProjectOperationUtils.tryProjectOperation; + +public class DSSProjectServiceImpl extends ServiceImpl implements DSSProjectService { + private static final Logger LOGGER = LoggerFactory.getLogger(DSSProjectServiceImpl.class); + @Autowired + private DSSProjectMapper projectMapper; + @Autowired + private DSSProjectUserService projectUserService; + @Autowired + private DSSProjectUserMapper projectUserMapper; + + public static final String MODE_SPLIT = ","; + public static final String KEY_SPLIT = "-"; + private final String SUPPORT_ABILITY = ProjectConf.SUPPORT_ABILITY.getValue(); + + @Override + public DSSProjectDO createProject(String username, ProjectCreateRequest projectCreateRequest) { + DSSProjectDO project = new DSSProjectDO(); + project.setName(projectCreateRequest.getName()); + project.setWorkspaceId(projectCreateRequest.getWorkspaceId()); + project.setCreateBy(username); + project.setUsername(username); + project.setCreateTime(new Date()); + project.setBusiness(projectCreateRequest.getBusiness()); + project.setProduct(projectCreateRequest.getProduct()); + project.setUpdateTime(new Date()); + project.setDescription(projectCreateRequest.getDescription()); + project.setApplicationArea(projectCreateRequest.getApplicationArea()); + //开发流程,编排模式组拼接 前后进行英文逗号接口 + project.setDevProcess(ProjectStringUtils.getModeStr(projectCreateRequest.getDevProcessList())); + project.setOrchestratorMode(ProjectStringUtils.getModeStr(projectCreateRequest.getOrchestratorModeList())); + projectMapper.insert(project); + return project; + } + + //修改dss_project工程字段 + @Override + public void modifyProject(String username, ProjectModifyRequest projectModifyRequest) throws DSSProjectErrorException { + //校验当前登录用户是否含有修改权限 +// projectUserService.isEditProjectAuth(projectModifyRequest.getId(), username); + DSSProjectDO project = new DSSProjectDO(); + //修改的字段 + project.setDescription(projectModifyRequest.getDescription()); + project.setUpdateTime(new Date()); + project.setUpdateByStr(username); + project.setDevProcess(ProjectStringUtils.getModeStr(projectModifyRequest.getDevProcessList())); + if (StringUtils.isNotBlank(projectModifyRequest.getApplicationArea())) { + project.setApplicationArea(Integer.valueOf(projectModifyRequest.getApplicationArea())); + } + project.setOrchestratorMode(ProjectStringUtils.getModeStr(projectModifyRequest.getOrchestratorModeList())); + project.setBusiness(projectModifyRequest.getBusiness()); + project.setProduct(projectModifyRequest.getProduct()); + + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", projectModifyRequest.getId()); + updateWrapper.eq("workspace_id", projectModifyRequest.getWorkspaceId()); + projectMapper.update(project, updateWrapper); + } + + /** + * 修改旧dss_project工程字段 + */ + @Override + public void modifyOldProject(DSSProjectDO updateProject, DSSProjectDO dbProject) { + DSSProjectDO project = new DSSProjectDO(); + //修改的字段 + project.setUpdateTime(new Date()); + project.setUpdateByStr(updateProject.getUpdateByStr()); + project.setDescription(updateProject.getDescription()); + project.setBusiness(updateProject.getBusiness()); + project.setApplicationArea(updateProject.getApplicationArea()); + + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", dbProject.getId()); + updateWrapper.eq("workspace_id", dbProject.getWorkspaceId()); + projectMapper.update(project, updateWrapper); + } + + @Override + public DSSProjectDO getProjectByName(String name) { + QueryWrapper projectQueryWrapper = new QueryWrapper<>(); + projectQueryWrapper.eq("name", name); + List projectList = projectMapper.selectList(projectQueryWrapper); + return CollectionUtils.isEmpty(projectList) ? null : projectList.get(0); + } + + @Override + public DSSProjectDO getProjectById(Long id) { + return projectMapper.selectById(id); + } + + @Override + public List getListByParam(ProjectQueryRequest projectRequest) { + //根据dss_project、dss_project_user查询出所在空间登录用户相关的工程 + List list; + if (isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername())) { + list = projectMapper.getListForAdmin(projectRequest); + } else { + list = projectMapper.getListByParam(projectRequest); + } + if (CollectionUtils.isEmpty(list)) { + return new ArrayList<>(); + } + + List projectResponseList = new ArrayList<>(); + for (QueryProjectVo projectVo : list) { + if (projectVo.getVisible() == 0) { + continue; + } + ProjectResponse projectResponse = new ProjectResponse(); + projectResponse.setApplicationArea(projectVo.getApplicationArea()); + projectResponse.setId(projectVo.getId()); + projectResponse.setBusiness(projectVo.getBusiness()); + projectResponse.setCreateBy(projectVo.getCreateBy()); + projectResponse.setDescription(projectVo.getDescription()); + projectResponse.setName(projectVo.getName()); + projectResponse.setProduct(projectVo.getProduct()); + projectResponse.setSource(projectVo.getSource()); + projectResponse.setArchive(projectVo.getArchive()); + projectResponse.setCreateTime(projectVo.getCreateTime()); + projectResponse.setUpdateTime(projectVo.getUpdateTime()); + projectResponse.setDevProcessList(ProjectStringUtils.convertList(projectVo.getDevProcess())); + projectResponse.setOrchestratorModeList(ProjectStringUtils.convertList(projectVo.getOrchestratorMode())); + projectResponseList.add(projectResponse); + + String pusername = projectVo.getPusername(); + String editPriv = projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_EDIT.getRank() + + KEY_SPLIT + projectRequest.getUsername(); + + Map> userPricMap = new HashMap<>(); + String[] tempstrArr = pusername.split(MODE_SPLIT); + + // 拆分有projectId +"-" + priv + "-" + username的拼接而成的字段, + // 从而得到:查看权限用户、编辑权限用户、发布权限用户 + for (String s : tempstrArr) { + String[] strArr = s.split(KEY_SPLIT); + if(strArr.length >= 3) { + String key = strArr[0] + KEY_SPLIT + strArr[1]; + userPricMap.computeIfAbsent(key, k -> new ArrayList<>()); + userPricMap.get(key).add(strArr[2]); + } + } + List accessUsers = userPricMap.get(projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_ACCESS.getRank()); + List editUsers = userPricMap.get(projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_EDIT.getRank()); + List releaseUsers = userPricMap.get(projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_RELEASE.getRank()); + projectResponse.setAccessUsers(CollectionUtils.isEmpty(accessUsers) ? new ArrayList<>() : accessUsers.stream().distinct().collect(Collectors.toList())); + projectResponse.setEditUsers(CollectionUtils.isEmpty(editUsers) ? new ArrayList<>() : editUsers.stream().distinct().collect(Collectors.toList())); + projectResponse.setReleaseUsers(CollectionUtils.isEmpty(releaseUsers) ? new ArrayList<>() : releaseUsers.stream().distinct().collect(Collectors.toList())); + + // 用户是否具有编辑权限 编辑权限和创建者都有 + if (!StringUtils.isEmpty(pusername) && + (pusername.contains(editPriv) || + projectVo.getCreateBy().equals(projectRequest.getUsername()) || + isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername())) || projectResponse.getReleaseUsers().contains(projectRequest.getUsername())) { + projectResponse.setEditable(true); + } else if (isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername()) || + projectVo.getCreateBy().equals(projectRequest.getUsername())) { + projectResponse.setEditable(true); + } else { + projectResponse.setEditable(false); + } + } + return projectResponseList; + } + + @Override + public ProjectInfoVo getProjectInfoById(Long id) { + return projectMapper.getProjectInfoById(id); + } + + + @Override + public void saveProjectRelation(DSSProjectDO project, Map projectMap) { + List relationPos = new ArrayList<>(); + projectMap.forEach((k, v) -> relationPos.add(new ProjectRelationPo(project.getId(), k.getId(), v))); + projectMapper.saveProjectRelation(relationPos); + } + + + @Override + public Long getAppConnProjectId(Long dssProjectId, String appConnName, List dssLabels) throws Exception { + AppConn appConn = AppConnManager.getAppConnManager().getAppConn(appConnName); + List appInstances = appConn.getAppDesc().getAppInstancesByLabels(dssLabels); + if (appInstances.get(0) != null) { + Long appInstanceId = appInstances.get(0).getId(); + return getAppConnProjectId(appInstanceId, dssProjectId); + } else { + LOGGER.error("the appInstances of AppConn {} is null.", appConnName); + return null; + } + } + + @Override + public Long getAppConnProjectId(Long appInstanceId, Long dssProjectId) { + return projectMapper.getAppConnProjectId(appInstanceId, dssProjectId); + } + + @Override + public void deleteProject(String username, ProjectDeleteRequest projectDeleteRequest, Workspace workspace) throws Exception { + DSSProjectDO dssProjectDO = projectMapper.selectById(projectDeleteRequest.getId()); + if (dssProjectDO == null) { + throw new DSSErrorException(600001, "工程不存在!"); + } + LOGGER.warn("user {} begins to delete project {} in workspace {}.", username, dssProjectDO.getName(), workspace.getWorkspaceName()); + if(!dssProjectDO.getUsername().equalsIgnoreCase(username)){ + throw new DSSErrorException(600002, "刪除工程失敗,沒有删除权限!" ); + } + if(projectDeleteRequest.isIfDelOtherSys()) { + LOGGER.warn("User {} requires to delete all projects with name {} in third-party AppConns.", username, dssProjectDO.getName()); + Map appInstanceToRefProjectId = new HashMap<>(10); + tryProjectOperation((appConn, appInstance) -> { + Long refProjectId = getAppConnProjectId(appInstance.getId(), projectDeleteRequest.getId()); + if(refProjectId == null) { + LOGGER.warn("delete project {} for third-party AppConn {} is ignored, appInstance is {}. Caused by: the refProjectId is null.", + dssProjectDO.getName(), appConn.getAppDesc().getAppName(), appInstance.getBaseUrl()); + return false; + } else { + appInstanceToRefProjectId.put(appInstance, refProjectId); + return true; + } + }, workspace, ProjectService::getProjectDeletionOperation, + null, + (appInstance, refProjectContentRequestRef) -> refProjectContentRequestRef.setProjectName(dssProjectDO.getName()) + .setRefProjectId(appInstanceToRefProjectId.get(appInstance)).setUserName(username), + (structureOperation, structureRequestRef) -> ((ProjectDeletionOperation) structureOperation).deleteProject((RefProjectContentRequestRef) structureRequestRef), + null, "delete refProject " + dssProjectDO.getName()); + } + projectMapper.deleteProject(projectDeleteRequest.getId()); + LOGGER.warn("User {} deleted project {}.", username, dssProjectDO.getName()); + } + + @Override + public List getProjectAbilities(String username) { + LOGGER.info("{} begins to get project ability", username); + return Arrays.asList(SUPPORT_ABILITY.trim().split(",")); + } + + @Override + public boolean isDeleteProjectAuth(Long projectId, String username) throws DSSProjectErrorException { + //校验当前登录用户是否含有删除权限,默认创建用户可以删除工程 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("id", projectId); + queryWrapper.eq("create_by", username); + long count = projectMapper.selectCount(queryWrapper); + if (count == 0) { + DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_NOT_EDIT_AUTH.getCode(), ProjectServerResponse.PROJECT_NOT_EDIT_AUTH.getMsg(), DSSProjectErrorException.class); + } + return true; + } + + private boolean isWorkspaceAdmin(Long workspaceId, String username) { + return !projectUserMapper.getUserWorkspaceAdminRole(workspaceId, username).isEmpty(); + } + + @Override + public List getDeletedProjects(ProjectQueryRequest projectRequest) { + //根据dss_project、dss_project_user查询出所在空间登录用户相关的工程,再删选出其中是已删除的项目 + List list = projectMapper.getDeletedProjects(projectRequest); + if (CollectionUtils.isEmpty(list)) { + return new ArrayList<>(); + } + List projectResponseList = new ArrayList<>(); + for (QueryProjectVo projectVo : list) { + ProjectResponse projectResponse = new ProjectResponse(); + projectResponse.setApplicationArea(projectVo.getApplicationArea()); + projectResponse.setId(projectVo.getId()); + projectResponse.setBusiness(projectVo.getBusiness()); + projectResponse.setCreateBy(projectVo.getCreateBy()); + projectResponse.setDescription(projectVo.getDescription()); + projectResponse.setName(projectVo.getName()); + projectResponse.setProduct(projectVo.getProduct()); + projectResponse.setSource(projectVo.getSource()); + projectResponse.setArchive(projectVo.getArchive()); + projectResponse.setCreateTime(projectVo.getCreateTime()); + projectResponse.setUpdateTime(projectVo.getUpdateTime()); + projectResponse.setDevProcessList(ProjectStringUtils.convertList(projectVo.getDevProcess())); + projectResponse.setOrchestratorModeList(ProjectStringUtils.convertList(projectVo.getOrchestratorMode())); + projectResponseList.add(projectResponse); + /** + * 拆分有projectId +"-" + priv + "-" + username的拼接而成的字段, + * 从而得到:查看权限用户、编辑权限用户、发布权限用户 + */ + String pusername = projectVo.getPusername(); + if (StringUtils.isEmpty(pusername)) { + continue; + } + Map> userPricMap = new HashMap<>(); + String[] tempstrArr = pusername.split(MODE_SPLIT); + + for (String s : tempstrArr) { + String[] strArr = s.split(KEY_SPLIT); + String key = strArr[0] + KEY_SPLIT + strArr[1]; + userPricMap.computeIfAbsent(key, k -> new ArrayList<>()); + userPricMap.get(key).add(strArr[2]); + } + List accessUsers = userPricMap.get(projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_ACCESS.getRank()); + List editUsers = userPricMap.get(projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_EDIT.getRank()); + List releaseUsers = userPricMap.get(projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_RELEASE.getRank()); + projectResponse.setAccessUsers(CollectionUtils.isEmpty(accessUsers) ? new ArrayList<>() : accessUsers.stream().distinct().collect(Collectors.toList())); + projectResponse.setEditUsers(CollectionUtils.isEmpty(editUsers) ? new ArrayList<>() : editUsers.stream().distinct().collect(Collectors.toList())); + projectResponse.setReleaseUsers(CollectionUtils.isEmpty(releaseUsers) ? new ArrayList<>() : releaseUsers.stream().distinct().collect(Collectors.toList())); + } + return projectResponseList; + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectUserServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectUserServiceImpl.java new file mode 100644 index 000000000..c6bb7c165 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectUserServiceImpl.java @@ -0,0 +1,200 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.framework.project.contant.ProjectServerResponse; +import com.webank.wedatasphere.dss.framework.project.dao.DSSProjectMapper; +import com.webank.wedatasphere.dss.framework.project.dao.DSSProjectUserMapper; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectUser; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; +import com.webank.wedatasphere.dss.framework.project.server.service.BMLService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectUserService; +import com.webank.wedatasphere.dss.framework.project.utils.ProjectUserUtils; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class DSSProjectUserServiceImpl implements DSSProjectUserService { + + protected final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + @Autowired + private DSSProjectUserMapper projectUserMapper; + @Autowired + @Qualifier("projectServerBMLService") + private BMLService bmlService; + @Autowired + private DSSProjectMapper dssProjectMapper; + + /** + * 是否有修改工程权限 + * + * @param projectId + * @param username + * @return + */ + @Override + public boolean isEditProjectAuth(Long projectId, String username) throws DSSProjectErrorException { + //校验当前登录用户是否含有修改权限 + QueryWrapper queryWrapper = new QueryWrapper(); + queryWrapper.eq("project_id", projectId); + queryWrapper.eq("username", username); + queryWrapper.ge("priv", ProjectUserPrivEnum.PRIV_EDIT.getRank());//编辑权限 + long count = projectUserMapper.selectCount(queryWrapper); + if (count == 0 && !isProjectOwner(projectId, username)) { + DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_NOT_EDIT_AUTH.getCode(), ProjectServerResponse.PROJECT_NOT_EDIT_AUTH.getMsg(), DSSProjectErrorException.class); + } + return true; + } + + /** + * 保存工程与用户关系 + * + * @param projectID + * @param request + */ + @Override + public void saveProjectUser(Long projectID, String username, ProjectCreateRequest request, Workspace workspace) throws Exception { + //將创建人默认为发布权限 + List releaseUsers = request.getReleaseUsers(); + //批量保存 + saveBatch(request.getWorkspaceId(), projectID, releaseUsers, request.getEditUsers(), request.getAccessUsers()); + //获取所有编辑权限的用户 + List sumEditUsers = ProjectUserUtils.getEditUserList(releaseUsers, request.getEditUsers()); + //去bml建一个工程 + bmlService.createBmlProject(username, request.getName(), sumEditUsers, request.getAccessUsers()); + } + + /** + * 修改工程与用户关系 + * + * @param dbProject + * @param modifyRequest + */ + @Override + public void modifyProjectUser(DSSProjectDO dbProject, ProjectModifyRequest modifyRequest, String loginuser, Workspace workspace) throws Exception { + projectUserMapper.deleteAllPriv(dbProject.getId()); + //將创建人默认为发布权限 + List releaseUsers = modifyRequest.getReleaseUsers(); + String username = dbProject.getUsername(); + //批量保存 + saveBatch(modifyRequest.getWorkspaceId(), dbProject.getId(), releaseUsers, modifyRequest.getEditUsers(), modifyRequest.getAccessUsers()); + + //获取所有编辑权限的用户 + List sumEditUsers = ProjectUserUtils.getEditUserList(releaseUsers, modifyRequest.getEditUsers()); + //更新底层的权限 + bmlService.updateProjectPriv(dbProject.getName(), username, sumEditUsers, modifyRequest.getAccessUsers()); + } + + + @Override + public List getListByParam(Long workspaceId, String username) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("workspace_id", workspaceId); + queryWrapper.eq("username", username); + queryWrapper.ge("priv", ProjectUserPrivEnum.PRIV_ACCESS.getRank()); + List DSSProjectUserList = projectUserMapper.selectList(queryWrapper); + return DSSProjectUserList; + } + + //批量保存 + public void saveBatch(Long workspaceId, Long projectID, List releaseUsers, List editUsers, List accessUsers) { + List realReleaseUsers = releaseUsers.stream().distinct().collect(Collectors.toList()); + List realEditUsers = editUsers.stream().distinct().collect(Collectors.toList()); + List realAccessUsers = accessUsers.stream().distinct().collect(Collectors.toList()); + List addList = new ArrayList<>(); + addList.addAll(ProjectUserUtils.createPUser(workspaceId, projectID, realReleaseUsers, ProjectUserPrivEnum.PRIV_RELEASE.getRank())); + addList.addAll(ProjectUserUtils.createPUser(workspaceId, projectID, realEditUsers, ProjectUserPrivEnum.PRIV_EDIT.getRank())); + addList.addAll(ProjectUserUtils.createPUser(workspaceId, projectID, realAccessUsers, ProjectUserPrivEnum.PRIV_ACCESS.getRank())); + if (CollectionUtils.isEmpty(addList)) { + return; + } + //分批插入 + List tempAddList = new ArrayList<>(); + for (int i = 0; i < addList.size(); i++) { + tempAddList.add(addList.get(i)); + if (i > 0 && i % 200 == 0) { + projectUserMapper.insertBatchProjectUser(tempAddList); + tempAddList.clear(); + } + } + if (tempAddList.size() > 0) { + projectUserMapper.insertBatchProjectUser(tempAddList); + } + } + + @Override + public boolean isAdminByUsername(Long workspaceId, String username) { + //管理员ID,这里写死 + int roleId = 1; + return projectUserMapper.isAdminByUsername(workspaceId, username, roleId).longValue() > 0; + } + + /** + * 根据用户名和工程id获取工程权限 + * + * @param projectId + * @param username + * @return + */ + @Override + public List getProjectUserPriv(Long projectId, String username) { + QueryWrapper queryWrapper = new QueryWrapper(); + queryWrapper.eq("project_id", projectId); + queryWrapper.eq("username", username); + return projectUserMapper.selectList(queryWrapper); + } + + @Override + public List getProjectPriv(Long projectId) { + return projectUserMapper.getPrivsByProjectId(projectId); + } + + private boolean isProjectOwner(Long projectId, String username) { + QueryWrapper queryWrapper = new QueryWrapper(); + queryWrapper.eq("id", projectId); + queryWrapper.eq("create_by", username); + return dssProjectMapper.selectCount(queryWrapper) > 0; + } + + @Override + public List listByPriv(Long projectId, ProjectUserPrivEnum privEnum) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.select("username").eq("project_id", projectId).ge("priv", privEnum.getRank()); + return projectUserMapper.selectList(queryWrapper); + } + + + @Override + public boolean isWorkspaceUser(Long workspaceId, String username) { + return projectUserMapper.isWorkspaceUser(workspaceId, username).longValue() > 0; + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ApplicationArea.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ApplicationArea.java new file mode 100644 index 000000000..a06799f60 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ApplicationArea.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.utils; + + +public enum ApplicationArea { + // Operational optimization, risk management, marketing recommendation, trend prediction, customer management + // 运营优化,风险管理,营销推荐,趋势预测,客户管理; + OO("运营优化","Operational Optimization",0), + RM("风险管理","Risk Management",1), + MR("营销推荐","Marketing Recommendation",2), + TP("趋势预测","Trend Prediction",3), + CM("客户管理","Customer Management",4); + + + private int id; + private String name; + private String enName; + + ApplicationArea(String name, String enName, int id) { + this.name = name; + this.enName = enName; + this.id = id; + } + + public String getName() { + return name; + } + + public int getId(){ + return this.id; + } + + public String getEnName(){ + return this.enName; + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/LambdaHelper.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/LambdaHelper.java new file mode 100644 index 000000000..234d1bdad --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/LambdaHelper.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.Consumer; + + + +public class LambdaHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(LambdaHelper.class); + + public static Consumer consumerWrapper(Consumer consumer){ + return t -> { + try{ + consumer.accept(t); + }catch(Exception e){ + LOGGER.error("Failed to do with {} ", t, e); + } + }; + } + + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectOperationUtils.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectOperationUtils.java new file mode 100644 index 000000000..8bf07bc2d --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectOperationUtils.java @@ -0,0 +1,64 @@ +package com.webank.wedatasphere.dss.framework.project.utils; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyStructureAppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.structure.StructureIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.utils.StructureOperationUtils; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import org.apache.commons.lang3.tuple.ImmutablePair; + +import java.util.function.*; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public class ProjectOperationUtils { + + public static void tryProjectOperation(BiPredicate isTryOperation, Workspace workspace, + Function getProjectOperation, + Consumer dssProjectContentRequestRefConsumer, + BiConsumer refProjectContentRequestRefConsumer, + BiFunction responseRefConsumer, + BiConsumer, V> dealResponseRefConsumer, + String errorMsg) { + // 排序,将 SchedulerAppConn 放在最前面进行操作,因为调度系统的操作最危险,如果有问题就第一个抛出来 + AppConnManager.getAppConnManager().listAppConns().stream().sorted((appConn1, appConn2) -> { + if(appConn1 instanceof SchedulerAppConn) { + return -1; + } else if(appConn2 instanceof SchedulerAppConn){ + return 1; + } else { + return 0; + } + }).filter(appConn -> appConn instanceof OnlyStructureAppConn).forEach(appConn -> { + StructureIntegrationStandard structureStandard = ((OnlyStructureAppConn) appConn).getOrCreateStructureStandard(); + appConn.getAppDesc().getAppInstances().forEach(appInstance -> { + if(isTryOperation == null || isTryOperation.test(appConn, appInstance)) { + V responseRef = StructureOperationUtils.tryProjectOperation(() -> structureStandard.getProjectService(appInstance), + getProjectOperation, + dssProjectContentRequestRefConsumer, + refProjectContentRequestRef -> refProjectContentRequestRefConsumer.accept(appInstance, refProjectContentRequestRef), + (structureOperation, structureRequestRef) -> { + structureRequestRef.setDSSLabels(appInstance.getLabels()).setWorkspace(workspace); + return responseRefConsumer.apply(structureOperation, (K) structureRequestRef); + }, errorMsg); + if(dealResponseRefConsumer != null) { + dealResponseRefConsumer.accept(new ImmutablePair<>(appConn, appInstance), responseRef); + } + } + }); + }); + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectStringUtils.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectStringUtils.java new file mode 100644 index 000000000..4bbe90273 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectStringUtils.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.utils; + + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.*; +import java.util.stream.Collectors; + +public class ProjectStringUtils { + + + private static final Logger LOGGER = LoggerFactory.getLogger(ProjectStringUtils.class); + + private static final String REDIRECT_FORMAT = "%s?redirect=%s&dssurl=${dssurl}&cookies=${cookies}"; + + public static final String MODE_SPLIT = ","; + public static final String KEY_SPLIT = "-"; + + public static List convertList(String str){ + if(org.apache.commons.lang.StringUtils.isEmpty(str)){ + return new ArrayList<>(); + } + return Arrays.stream(str.split(MODE_SPLIT)).map(String::trim).filter((s)-> StringUtils.isNotBlank(s)).distinct().collect(Collectors.toList()); + } + + //拼接 前后使用英文逗号结尾 + public static String getModeStr(List strList){ + if(CollectionUtils.isEmpty(strList)){ + return null; + } + return strList.stream().map(String::trim).filter((s)-> StringUtils.isNotBlank(s)).distinct().collect(Collectors.joining(MODE_SPLIT,MODE_SPLIT,MODE_SPLIT)); + } + + private static String URLEndoder(String str){ + try { + return URLEncoder.encode(str,"utf-8"); + } catch (UnsupportedEncodingException e) { + LOGGER.warn("endoe failed:",e); + return str; + } + } + + public static String redirectUrlFormat(String redirectUrl,String url){ + return String.format(REDIRECT_FORMAT,redirectUrl,URLEndoder(url)); + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectUserUtils.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectUserUtils.java new file mode 100644 index 000000000..2fe2ac253 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectUserUtils.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.utils; + +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectUser; +import org.apache.commons.collections.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class ProjectUserUtils { + + + public static List createPUser(Long wid, Long pid, List users, int priv){ + List retList = new ArrayList<>(); + if(CollectionUtils.isEmpty(users)){ + return retList; + } + users.forEach(tname->{ + DSSProjectUser DSSProjectUser = new DSSProjectUser(wid, pid, tname,priv); + retList.add(DSSProjectUser); + }); + return retList; + } + + + public static List getEditUserList(List releaseUsers, List editUsers){ + List sumEditUsers = new ArrayList<>(); + if (!CollectionUtils.isEmpty(releaseUsers)) { + sumEditUsers.addAll(releaseUsers); + } + if (!CollectionUtils.isEmpty(editUsers)) { + sumEditUsers.addAll(editUsers); + } + return sumEditUsers.stream().distinct().collect(Collectors.toList()); + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/resources/application-dss.yml b/dss-framework/dss-framework-project-server/src/main/resources/application-dss.yml new file mode 100644 index 000000000..b0473838d --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/resources/application-dss.yml @@ -0,0 +1,23 @@ + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + #instance: + #prefer-ip-address: true + #instance-id: ${spring.cloud.client.ip-address}:${server.port} + #metadata-map: + #test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + diff --git a/dss-framework/dss-framework-project-server/src/main/resources/dss-framework-project-server.properties b/dss-framework/dss-framework-project-server/src/main/resources/dss-framework-project-server.properties new file mode 100644 index 000000000..d98dbc97f --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/resources/dss-framework-project-server.properties @@ -0,0 +1,39 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9202 +spring.spring.application.name=dss-framework-project-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.framework.workspace.restful,com.webank.wedatasphere.dss.framework.project.restful,com.webank.wedatasphere.dss.framework.release.restful,com.webank.wedatasphere.dss.framework.dbapi.restful,com.webank.wedatasphere.dss.framework.admin.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/workspace/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/application/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/project/dao/impl/*Mapper.xml,classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/release/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/admin/xml/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/dbapi/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.application.entity,com.webank.wedatasphere.dss.common.entity,com.webank.wedatasphere.dss.framework.workspace.bean,com.webank.wedatasphere.dss.framework.project.entity,com.webank.wedatasphere.dss.framework.appconn.entity,com.webank.wedatasphere.dss.framework.release.entity,com.webank.wedatasphere.dss.framework.admin.pojo.entity,com.webank.wedatasphere.dss.framework.dbapi.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.framework.workspace.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.framework.project.dao,com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.framework.release.dao,com.webank.wedatasphere.dss.framework.admin.xml,com.webank.wedatasphere.dss.framework.dbapi.dao + +#wds.linkis.gateway.ip=127.0.0.1 +#wds.linkis.gateway.port=9001 +#wds.linkis.gateway.url=http://127.0.0.1:9001/ diff --git a/dss-framework/dss-framework-project-server/src/main/resources/dss.properties b/dss-framework/dss-framework-project-server/src/main/resources/dss.properties new file mode 100644 index 000000000..753837afb --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/resources/dss.properties @@ -0,0 +1,53 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +wds.linkis.gateway.ip=127.0.0.1 +wds.linkis.gateway.port=9001 +wds.linkis.gateway.url=http://127.0.0.1:9001/ + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/dss_dev_1?characterEncoding=UTF-8 + +wds.linkis.server.mybatis.datasource.username= +wds.linkis.server.mybatis.datasource.password= + +spring.server.port=9202 +spring.spring.application.name=dss-framework-project-server + + +wds.linkis.reflect.scan.package=org.apache.linkis,com.webank.wedatasphere.dss +spring.spring.mvc.servlet.path=/api/rest_j/v1 +spring.spring.servlet.multipart.max-file-size=200MB +spring.spring.servlet.multipart.max-request-size=200MB + + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.framework.workspace.restful,com.webank.wedatasphere.dss.framework.project.restful,com.webank.wedatasphere.dss.framework.release.restful,com.webank.wedatasphere.dss.framework.dbapi.restful,com.webank.wedatasphere.dss.framework.admin.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/workspace/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/application/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/project/dao/impl/*Mapper.xml,classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/release/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/admin/xml/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/dbapi/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.application.entity,com.webank.wedatasphere.dss.common.entity,com.webank.wedatasphere.dss.framework.workspace.bean,com.webank.wedatasphere.dss.framework.project.entity,com.webank.wedatasphere.dss.framework.appconn.entity,com.webank.wedatasphere.dss.framework.release.entity,com.webank.wedatasphere.dss.framework.admin.pojo.entity,com.webank.wedatasphere.dss.framework.dbapi.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.framework.workspace.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.framework.project.dao,com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.framework.release.dao,com.webank.wedatasphere.dss.framework.admin.xml,com.webank.wedatasphere.dss.framework.dbapi.dao + +wds.linkis.server.user.restful.uri.pass.auth=/api/rest_j/v1/dss/framework/audit/script/download/save,/api/rest_j/v1/dss/framework/audit/script/download/query,/api/rest_j/v1/dss/framework/exchangis/project/tree,/api/rest_j/v1/dss/framework/exchangis/task/tree,/api/rest_j/v1/dss/framework/exchangis/shell + diff --git a/dss-framework/dss-framework-project-server/src/main/resources/log4j.properties b/dss-framework/dss-framework-project-server/src/main/resources/log4j.properties new file mode 100644 index 000000000..3f4c83b72 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/resources/log4j.properties @@ -0,0 +1,38 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/dss-apiservice-server.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/resources/log4j2.xml b/dss-framework/dss-framework-project-server/src/main/resources/log4j2.xml new file mode 100644 index 000000000..c00e1eaa7 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/resources/log4j2.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-framework/dss-framework-project-server/src/main/resources/token.properties b/dss-framework/dss-framework-project-server/src/main/resources/token.properties new file mode 100644 index 000000000..a52a8983d --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/resources/token.properties @@ -0,0 +1,19 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +${userName}=${password} \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/DSSProjectServerApplication.scala b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/DSSProjectServerApplication.scala new file mode 100644 index 000000000..578d7464c --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/DSSProjectServerApplication.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.server +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + + +object DSSProjectServerApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiver.scala b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiver.scala new file mode 100644 index 000000000..4dd0d108a --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiver.scala @@ -0,0 +1,93 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.server.rpc + +import java.util + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject +import com.webank.wedatasphere.dss.common.protocol.{ProxyUserCheckRequest, ResponseProxyUserCheck} +import com.webank.wedatasphere.dss.common.protocol.project.{ProjectInfoRequest, ProjectRefIdRequest, ProjectRefIdResponse, ProjectRelationRequest, ProjectRelationResponse, ProjectUserAuthRequest, ProjectUserAuthResponse} +import com.webank.wedatasphere.dss.framework.admin.service.DssProxyUserService +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO +import com.webank.wedatasphere.dss.framework.project.entity.vo.ProjectInfoVo +import com.webank.wedatasphere.dss.framework.project.service.{DSSProjectService, DSSProjectUserService} +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService +import org.apache.linkis.protocol.usercontrol.{RequestUserListFromWorkspace, RequestUserWorkspace, ResponseUserWorkspace, ResponseWorkspaceUserList} +import org.apache.linkis.rpc.{Receiver, Sender} +import org.springframework.beans.BeanUtils +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ +import scala.concurrent.duration.Duration + + +@Component +class ProjectReceiver(projectService: DSSProjectService, + dssWorkspaceUserService: DSSWorkspaceUserService, + projectUserService: DSSProjectUserService, + dssProxyUserService: DssProxyUserService) extends Receiver { + + override def receive(message: Any, sender: Sender): Unit = { + + } + + override def receiveAndReply(message: Any, sender: Sender): Any = { + message match { + case projectRelationRequest: ProjectRelationRequest => + val dssProjectId = projectRelationRequest.getDssProjectId + val dssLabels = projectRelationRequest.getDssLabels + val appConnName = projectRelationRequest.getAppconnName + val appConnProjectId = projectService.getAppConnProjectId(dssProjectId, appConnName, dssLabels) + new ProjectRelationResponse(dssProjectId, appConnName, dssLabels, appConnProjectId) + case projectRefIdRequest: ProjectRefIdRequest => + val refProjectId = projectService.getAppConnProjectId(projectRefIdRequest.getAppInstanceId, projectRefIdRequest.getDssProjectId); + new ProjectRefIdResponse(projectRefIdRequest.getAppInstanceId, projectRefIdRequest.getDssProjectId, refProjectId); + case requestUserWorkspace: RequestUserWorkspace => + val userWorkspaceIds: util.List[Integer] = dssWorkspaceUserService.getUserWorkspaceIds(requestUserWorkspace.getUserName) + new ResponseUserWorkspace(userWorkspaceIds) + + case requestUserListFromWorkspace: RequestUserListFromWorkspace => + val userList = requestUserListFromWorkspace.getUserWorkspaceIds.flatMap(id => dssWorkspaceUserService.getAllWorkspaceUsers(id)).distinct + new ResponseWorkspaceUserList(userList) + + case projectInfoRequest: ProjectInfoRequest => + val dssProjectDO: DSSProjectDO = projectService.getProjectById(projectInfoRequest.getProjectId) + val projectInfoVo: ProjectInfoVo = projectService.getProjectInfoById(projectInfoRequest.getProjectId) + val DSSProject = new DSSProject() + BeanUtils.copyProperties(dssProjectDO, DSSProject) + DSSProject.setWorkspaceName(projectInfoVo.getWorkspaceName) + DSSProject + + case projectUserAuthRequest: ProjectUserAuthRequest => { + val projectId = projectUserAuthRequest.getProjectId + val userName = projectUserAuthRequest.getUserName + val projectDo: DSSProjectDO = projectService.getProjectById(projectId) + val privList = projectUserService.getProjectUserPriv(projectId, userName).map(_.getPriv) + new ProjectUserAuthResponse(projectId, userName, privList, projectDo.getCreateBy) + } + case proxyUserCheckRequest: ProxyUserCheckRequest => { + val isExists = dssProxyUserService.isExists(proxyUserCheckRequest.userName, proxyUserCheckRequest.proxyUser) + ResponseProxyUserCheck(isExists,dssProxyUserService.getProxyUserNameList(proxyUserCheckRequest.userName)) + } + } + } + + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = { + null + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiverChooser.scala b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiverChooser.scala new file mode 100644 index 000000000..9841c0700 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiverChooser.scala @@ -0,0 +1,66 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.server.rpc + +import com.webank.wedatasphere.dss.common.protocol.ProxyUserCheckRequest +import com.webank.wedatasphere.dss.common.protocol.project.{ProjectInfoRequest, ProjectRelationRequest} +import com.webank.wedatasphere.dss.framework.admin.service.DssProxyUserService +import com.webank.wedatasphere.dss.framework.project.service.{DSSProjectService, DSSProjectUserService} +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService +import com.webank.wedatasphere.dss.orchestrator.common.protocol.{RequestProjectImportOrchestrator, RequestProjectUpdateOrcVersion} +import javax.annotation.PostConstruct +import org.apache.linkis.protocol.usercontrol.{RequestUserListFromWorkspace, RequestUserWorkspace} +import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + + +@Component +class ProjectReceiverChooser extends ReceiverChooser { + + @Autowired + private var projectService: DSSProjectService = _ + + @Autowired + private var dssWorkspaceUserService: DSSWorkspaceUserService = _ + + @Autowired + private var dssProjectUserService: DSSProjectUserService = _ + + @Autowired + private var dssProxyUserService : DssProxyUserService= _ + + private var receiver: Option[ProjectReceiver] = _ + + @PostConstruct + def init(): Unit = receiver = Some(new ProjectReceiver(projectService, dssWorkspaceUserService, dssProjectUserService,dssProxyUserService)) + + override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = event.message match { + case _: ProjectRelationRequest => receiver + case _: RequestUserWorkspace => receiver + case _: RequestUserListFromWorkspace => receiver + case _: ProjectInfoRequest => receiver + case _: RequestProjectImportOrchestrator => receiver + case _: RequestProjectUpdateOrcVersion => receiver + case _: ProxyUserCheckRequest => receiver + case _ => None + } +} + + + + diff --git a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/service/BMLService.scala b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/service/BMLService.scala new file mode 100644 index 000000000..f7e946c62 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/service/BMLService.scala @@ -0,0 +1,183 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.server.service + +import java.io.{ByteArrayInputStream, InputStream} +import java.util +import java.util.UUID + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException +import com.webank.wedatasphere.dss.common.utils.IoUtils +import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory} +import org.apache.linkis.bml.protocol.{BmlDownloadResponse, BmlUpdateResponse, BmlUploadResponse} +import org.apache.linkis.common.utils.{JavaLog, Logging, Utils} +import org.apache.commons.io.IOUtils +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +@Component("projectServerBMLService") +class BMLService extends Logging{ + + def upload(userName: String, content: String, fileName: String, projectName:String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = createBMLClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def upload(userName: String, inputStream: InputStream, fileName: String, projectName:String): util.Map[String, Object] = { + val client: BmlClient = createBMLClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def update(userName: String, resourceId: String, inputStream: InputStream): util.Map[String, Object] = { + val client: BmlClient = createBMLClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, "", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def update(userName: String, resourceId: String, content: String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = createBMLClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, UUID.randomUUID().toString+".json", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def query(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = createBMLClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "string" -> inputstremToString(resource.inputStream) + } + + def download(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = createBMLClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "is" -> resource.inputStream + } + + def downloadToLocalPath(userName: String, resourceId: String, version: String, path: String): String = { + val result = download(userName,resourceId,version) + val is = result.get("is").asInstanceOf[InputStream] + val os = IoUtils.generateExportOutputStream(path) + Utils.tryFinally(IOUtils.copy(is,os)){ + IOUtils.closeQuietly(os) + IOUtils.closeQuietly(is) + } + path + } + + def downloadAndGetFlowJson(userName: String, resourceId: String, version: String, path: String): String = { + downloadToLocalPath(userName,resourceId,version,path) + //因为下载到指定目录后返回的resource的Stream为null,只能从文件重新读取。 + val is = IoUtils.generateInputInputStream(path) + Utils.tryFinally(inputstremToString(is))(IOUtils.closeQuietly(is)) + } + + def readLocalResourceFile(userName: String,readPath: String): InputStream ={ + IoUtils.generateInputInputStream(readPath) + } + + def readLocalFlowJsonFile(userName: String,readPath: String): String ={ + var inputStream:InputStream = null + var flowJson:String = null + Utils.tryFinally{ + inputStream = IoUtils.generateInputInputStream(readPath) + flowJson = inputstremToString(inputStream) + }{ + IOUtils.closeQuietly(inputStream) + } + flowJson + } + + private def inputstremToString(inputStream: InputStream): String = { + scala.io.Source.fromInputStream(inputStream).mkString + } + + private def createBMLClient(userName: String): BmlClient = { + if (userName == null) + BmlClientFactory.createBmlClient() + else + BmlClientFactory.createBmlClient(userName) + } + + def createBmlProject(username:String, projectName:String, editUsers:util.List[String], + accessUsers:util.List[String] ): Unit ={ + val client = createBMLClient(username) + val response = client.createBmlProject(username, projectName, accessUsers, editUsers) + if (response.isSuccess){ + logger.info(s"for user $username create bml project $projectName success") + }else{ + logger.error(s"for user $username create bml project $projectName failed") + } + } + + def attachResourceAndProject(username:String, projectName:String, resourceId:String):Unit = { + val client = createBMLClient(username) + val response = client.attachResourceAndProject(projectName, resourceId) + if (response.isSuccess){ + logger.info(s"attach $username and $projectName success") + }else{ + logger.error(s"attach $username and $projectName failed") + } + } + + + def updateProjectPriv(projectName:String, username:String, editUsers:util.List[String], + accessUsers:util.List[String]): Unit ={ + val client = createBMLClient(username) + val response = client.updateProjectPriv(username, projectName, editUsers, accessUsers) + if (response.isSuccess){ + logger.info(s"attach $username and $projectName success") + }else{ + logger.error(s"attach $username and $projectName failed") + } + } + + + +} diff --git a/dss-framework/dss-framework-workspace-server/bin/start-dss-framework-workspace-server.sh b/dss-framework/dss-framework-workspace-server/bin/start-dss-framework-workspace-server.sh new file mode 100644 index 000000000..8b875b08c --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/bin/start-dss-framework-workspace-server.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` +export DWS_ENGINE_MANAGER_HOME=$HOME + +export DWS_ENGINE_MANAGER_PID=$HOME/bin/linkis.pid + +if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then + pid=$(cat ${DWS_ENGINE_MANAGER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "DSS SERVER is already running." + return 0; + fi +fi + +export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" + +export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=20014" + +#export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" + +nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* org.apache.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "DSS SERVER start failed!" + sleep 1 + exit 1 +else + echo "DSS SERVER start succeeded!" + echo $pid > $DWS_ENGINE_MANAGER_PID + sleep 1 +fi +exit 1 \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/bin/stop-dss-framework-workspace-server.sh b/dss-framework/dss-framework-workspace-server/bin/stop-dss-framework-workspace-server.sh new file mode 100644 index 000000000..2ca464de6 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/bin/stop-dss-framework-workspace-server.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export FLOW_ENTRANCE_HOME_PID=$HOME/bin/linkis.pid + +function wait_for_FLOW_ENTRANCE_HOME_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${FLOW_ENTRANCE_HOME_PID}" ]]; then + echo "DSS SERVER is not running" +else + pid=$(cat ${FLOW_ENTRANCE_HOME_PID}) + if [[ -z "${pid}" ]]; then + echo "DSS SERVER is not running" + else + wait_for_FLOW_ENTRANCE_HOME_to_die $pid 40 + $(rm -f ${FLOW_ENTRANCE_HOME_PID}) + echo "DSS SERVER is stopped." + fi +fi \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/pom.xml b/dss-framework/dss-framework-workspace-server/pom.xml new file mode 100644 index 000000000..289248213 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/pom.xml @@ -0,0 +1,139 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + + 4.0.0 + + dss-framework-workspace-server + + + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + provided + + + org.apache.linkis + linkis-httpclient + ${linkis.version} + + + linkis-common + org.apache.linkis + + + + + + com.webank.wedatasphere.dss + dss-framework-common + ${dss.version} + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + + org.apache.commons + commons-math3 + provided + + + com.webank.wedatasphere.dss + dss-appconn-manager-client + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-framework-admin-service + ${dss.version} + provided + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/assembly/distribution.xml b/dss-framework/dss-framework-workspace-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..6abb63037 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/assembly/distribution.xml @@ -0,0 +1,320 @@ + + + + dss-framework-workspace-server + + zip + + true + dss-framework-workspace-server + + + + + + lib + true + true + false + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/annotation/DSSWorkspaceUM.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/annotation/DSSWorkspaceUM.java new file mode 100644 index 000000000..74e80506d --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/annotation/DSSWorkspaceUM.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface DSSWorkspaceUM { + String value() default ""; +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/annotation/WorkspaceIdChecker.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/annotation/WorkspaceIdChecker.java new file mode 100644 index 000000000..3de0c1ef9 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/annotation/WorkspaceIdChecker.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface WorkspaceIdChecker { + String value() default ""; +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSPrivCheckerAspect.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSPrivCheckerAspect.java new file mode 100644 index 000000000..ff8e4b818 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSPrivCheckerAspect.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.aop; + +import org.aspectj.lang.annotation.Aspect; +import org.springframework.stereotype.Component; + + +@Aspect +@Component +public class DSSPrivCheckerAspect { + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSUWorkspaceIdCheckerAspect.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSUWorkspaceIdCheckerAspect.java new file mode 100644 index 000000000..3f09b9f16 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSUWorkspaceIdCheckerAspect.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.aop; + +import org.aspectj.lang.annotation.Aspect; +import org.springframework.stereotype.Component; + + +@Aspect +@Component +public class DSSUWorkspaceIdCheckerAspect { + + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSWorkspaceUMAspect.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSWorkspaceUMAspect.java new file mode 100644 index 000000000..e42e2b6a9 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/aop/DSSWorkspaceUMAspect.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.aop; + +import org.aspectj.lang.annotation.Aspect; +import org.springframework.stereotype.Component; + + +@Aspect +@Component +public class DSSWorkspaceUMAspect { +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/BaseEntity.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/BaseEntity.java new file mode 100644 index 000000000..0f9960560 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/BaseEntity.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import java.util.Date; + + +public class BaseEntity { + + private String createBy; + + private Date createTime = new Date(); + + private Date lastUpdateTime = new Date(); + + private String lastUpdateUser; + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(Date lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public String getLastUpdateUser() { + return lastUpdateUser; + } + + public void setLastUpdateUser(String lastUpdateUser) { + this.lastUpdateUser = lastUpdateUser; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSApplication.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSApplication.java new file mode 100644 index 000000000..e69de29bb diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSApplicationBean.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSApplicationBean.java new file mode 100644 index 000000000..548f378bc --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSApplicationBean.java @@ -0,0 +1,76 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + + +public class DSSApplicationBean { + private Integer id; + private String name; + private String url; + private String homepageUri; + private String label; + + public DSSApplicationBean() { + } + + public DSSApplicationBean(String name, String url, String homepageUri, String label) { + this.name = name; + this.url = url; + this.homepageUri = homepageUri; + this.label = label; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getHomepageUri() { + return homepageUri; + } + + public void setHomepageUri(String homepageUri) { + this.homepageUri = homepageUri; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSComponentRole.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSComponentRole.java new file mode 100644 index 000000000..e6d7da526 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSComponentRole.java @@ -0,0 +1,102 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.io.Serializable; +import java.util.Date; + + +@TableName(value = "dss_workspace_appconn_role") +public class DSSComponentRole implements Serializable { + private static final long serialVersionUID=1L; + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + private Long workspaceId; + + private Integer componentId; + + private Integer roleId; + + private Integer priv; + + private Date updateTime; + + private String updateby; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Integer getComponentId() { + return componentId; + } + + public void setComponentId(Integer componentId) { + this.componentId = componentId; + } + + public Integer getRoleId() { + return roleId; + } + + public void setRoleId(Integer roleId) { + this.roleId = roleId; + } + + public Integer getPriv() { + return priv; + } + + public void setPriv(Integer priv) { + this.priv = priv; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getUpdateby() { + return updateby; + } + + public void setUpdateby(String updateby) { + this.updateby = updateby; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSDictionary.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSDictionary.java new file mode 100644 index 000000000..d8305f7a9 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSDictionary.java @@ -0,0 +1,304 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.io.Serializable; +import java.util.Date; + + +@TableName(value = "dss_workspace_dictionary") +public class DSSDictionary implements Serializable { + + private static final long serialVersionUID=1L; + + /** + * 主键ID,默认为0,所有空间都有 + */ + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + + /** + * 空间ID,默认为0,所有空间都有 + */ + private Integer workspaceId; + + /** + * 父key + */ + private String parentKey; + + /** + * 名称 + */ + private String dicName; + /** + * 名称(英文) + */ + private String dicNameEn; + + /** + * key 相当于编码,空间是w_开头,工程是p_ + */ + private String dicKey; + + /** + * key对应的值 + */ + private String dicValue; + + /** + * key对应的值(英文) + */ + private String dicValueEn; + + /** + * 标题 + */ + private String title; + + /** + * 标题(英文) + */ + private String titleEn; + + /** + * url + */ + private String url; + + /** + * url类型: 0-内部系统,1-外部系统;默认是内部 + */ + private Integer urlType; + + /** + * 图标 + */ + private String icon; + + /** + * 序号 + */ + private Integer orderNum; + + /** + * 备注 + */ + private String remark; + + /** + * 创建人 + */ + private String createUser; + + /** + * 创建时间 + */ + private Date createTime; + + /** + * 更新人 + */ + private String updateUser; + + /** + * 更新时间 + */ + private Date updateTime; + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public String getParentKey() { + return parentKey; + } + + public void setParentKey(String parentKey) { + this.parentKey = parentKey; + } + + public String getDicName() { + return dicName; + } + + public void setDicName(String dicName) { + this.dicName = dicName; + } + + public String getDicKey() { + return dicKey; + } + + public void setDicKey(String dicKey) { + this.dicKey = dicKey; + } + + public String getDicValue() { + return dicValue; + } + + public void setDicValue(String dicValue) { + this.dicValue = dicValue; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public Integer getUrlType() { + return urlType; + } + + public void setUrlType(Integer urlType) { + this.urlType = urlType; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public Integer getOrderNum() { + return orderNum; + } + + public void setOrderNum(Integer orderNum) { + this.orderNum = orderNum; + } + + public String getRemark() { + return remark; + } + + public void setRemark(String remark) { + this.remark = remark; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getDicNameEn() { + return dicNameEn; + } + + public void setDicNameEn(String dicNameEn) { + this.dicNameEn = dicNameEn; + } + + public String getDicValueEn() { + return dicValueEn; + } + + public void setDicValueEn(String dicValueEn) { + this.dicValueEn = dicValueEn; + } + + public String getTitleEn() { + return titleEn; + } + + public void setTitleEn(String titleEn) { + this.titleEn = titleEn; + } + + @Override + public String toString() { + return "DSSDictionary{" + + "id=" + id + + ", workspaceId=" + workspaceId + + ", parentKey='" + parentKey + '\'' + + ", dicName='" + dicName + '\'' + + ", dicNameEn='" + dicNameEn + '\'' + + ", dicKey='" + dicKey + '\'' + + ", dicValue='" + dicValue + '\'' + + ", dicValueEn='" + dicValueEn + '\'' + + ", title='" + title + '\'' + + ", titleEn='" + titleEn + '\'' + + ", url='" + url + '\'' + + ", urlType=" + urlType + + ", icon='" + icon + '\'' + + ", orderNum=" + orderNum + + ", remark='" + remark + '\'' + + ", createUser='" + createUser + '\'' + + ", createTime=" + createTime + + ", updateUser='" + updateUser + '\'' + + ", updateTime=" + updateTime + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSFavorite.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSFavorite.java new file mode 100644 index 000000000..e73c3804f --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSFavorite.java @@ -0,0 +1,79 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.framework.workspace.bean; + + +public class DSSFavorite extends BaseEntity { + + private Long id; + + private String username; + + private Long workspaceId; + + private Long menuAppConnId; + + private Integer order; + private String type; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Long getMenuAppConnId() { + return menuAppConnId; + } + + public void setMenuAppConnId(Long menuApplicationId) { + this.menuAppConnId = menuApplicationId; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSHomepage.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSHomepage.java new file mode 100644 index 000000000..48ab10de3 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSHomepage.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + + +public class DSSHomepage { + private int id; + private String name; + private String url; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSMenuRole.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSMenuRole.java new file mode 100644 index 000000000..0a00b471d --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSMenuRole.java @@ -0,0 +1,115 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.io.Serializable; +import java.util.Date; + + +@TableName(value = "dss_workspace_menu_role") +public class DSSMenuRole implements Serializable { + + private static final long serialVersionUID=1L; + + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + + private Integer workspaceId; + + private Integer menuId; + + private Integer roleId; + + private Integer priv; + + private Date updateTime; + + private String updateby; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public Integer getMenuId() { + return menuId; + } + + public void setMenuId(Integer menuId) { + this.menuId = menuId; + } + + public Integer getRoleId() { + return roleId; + } + + public void setRoleId(Integer roleId) { + this.roleId = roleId; + } + + public Integer getPriv() { + return priv; + } + + public void setPriv(Integer priv) { + this.priv = priv; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getUpdateby() { + return updateby; + } + + public void setUpdateby(String updateby) { + this.updateby = updateby; + } + + @Override + public String toString() { + return "DssMenuRole{" + + "id=" + id + + ", workspaceId=" + workspaceId + + ", menuId=" + menuId + + ", roleId=" + roleId + + ", priv=" + priv + + ", updateTime=" + updateTime + + ", updateby='" + updateby + '\'' + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspace.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspace.java new file mode 100644 index 000000000..a8248371f --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspace.java @@ -0,0 +1,133 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import java.util.Date; + + +public class DSSWorkspace { + private int id; + private String name; + private String label; + private String description; + private String createBy; + private Date createTime; + private String department; + private String product; + private String source; + private Date lastUpdateTime; + private String lastUpdateUser; + private String workspaceType; + + public String getWorkspaceType() { + return workspaceType; + } + + public void setWorkspaceType(String workspaceType) { + this.workspaceType = workspaceType; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public Date getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(Date lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public String getLastUpdateUser() { + return lastUpdateUser; + } + + public void setLastUpdateUser(String lastUpdateUser) { + this.lastUpdateUser = lastUpdateUser; + } + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceComponentPriv.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceComponentPriv.java new file mode 100644 index 000000000..3b05f0037 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceComponentPriv.java @@ -0,0 +1,100 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import java.util.Date; + + +public class DSSWorkspaceComponentPriv { + + private int id; + private int workspaceId; + private int roleId; + private int componentId; + private int priv; + private Date updateTime; + private String updateBy; + + + public DSSWorkspaceComponentPriv() { + } + + public DSSWorkspaceComponentPriv(int workspaceId, int roleId, int componentId, int priv, Date updateTime, String updateBy) { + this.workspaceId = workspaceId; + this.roleId = roleId; + this.componentId = componentId; + this.priv = priv; + this.updateTime = updateTime; + this.updateBy = updateBy; + } + + public int getComponentId() { + return componentId; + } + + public void setComponentId(int componentId) { + this.componentId = componentId; + } + + public int getPriv() { + return priv; + } + + public void setPriv(int priv) { + this.priv = priv; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public int getRoleId() { + return roleId; + } + + public void setRoleId(int roleId) { + this.roleId = roleId; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceComponentRolePriv.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceComponentRolePriv.java new file mode 100644 index 000000000..87fdd4a7e --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceComponentRolePriv.java @@ -0,0 +1,106 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.google.common.base.Objects; + +import java.util.Date; + + +public class DSSWorkspaceComponentRolePriv { + private Integer id; + private Integer workspaceId; + private Integer componentId; + private Integer roleId; + private Integer priv; + private Date lastUpdateTime; + private String updateBy; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public Integer getComponentId() { + return componentId; + } + + public void setComponentId(Integer componentId) { + this.componentId = componentId; + } + + public Integer getRoleId() { + return roleId; + } + + public void setRoleId(Integer roleId) { + this.roleId = roleId; + } + + public Integer getPriv() { + return priv; + } + + public void setPriv(Integer priv) { + this.priv = priv; + } + + public Date getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(Date lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DSSWorkspaceComponentRolePriv that = (DSSWorkspaceComponentRolePriv) o; + return Objects.equal(componentId, that.componentId) && + Objects.equal(roleId, that.roleId); + } + + @Override + public int hashCode() { + return Objects.hashCode(componentId, roleId); + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceHomepage.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceHomepage.java new file mode 100644 index 000000000..22e218cd5 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceHomepage.java @@ -0,0 +1,88 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.time.LocalDateTime; +import java.util.Date; + + +@TableName(value = "dss_workspace_homepage") +public class DSSWorkspaceHomepage { + private static final long serialVersionUID=1L; + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + private Integer workspaceId; + private Integer roleId; + private String homepageUrl; + private Date updateTime; + + + public DSSWorkspaceHomepage(){ + + } + + public DSSWorkspaceHomepage(Integer workspaceId, Integer roleId, String homepageUrl, Date updateTime) { + this.workspaceId = workspaceId; + this.roleId = roleId; + this.homepageUrl = homepageUrl; + this.updateTime = updateTime; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public Integer getRoleId() { + return roleId; + } + + public void setRoleId(Integer roleId) { + this.roleId = roleId; + } + + public String getHomepageUrl() { + return homepageUrl; + } + + public void setHomepageUrl(String homepageUrl) { + this.homepageUrl = homepageUrl; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceHomepageSetting.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceHomepageSetting.java new file mode 100644 index 000000000..899741f72 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceHomepageSetting.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import java.util.Date; + + +public class DSSWorkspaceHomepageSetting { + private Integer id; + private Integer workspaceId; + private Integer roleId; + private String homepageUrl; + private Date updateTime; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public Integer getRoleId() { + return roleId; + } + + public void setRoleId(Integer roleId) { + this.roleId = roleId; + } + + public String getHomepageUrl() { + return homepageUrl; + } + + public void setHomepageUrl(String homepageUrl) { + this.homepageUrl = homepageUrl; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenu.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenu.java new file mode 100644 index 000000000..90753a1c5 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenu.java @@ -0,0 +1,118 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.baomidou.mybatisplus.annotation.TableName; + +import java.util.Date; + +@TableName(value = "dss_workspace_menu") +public class DSSWorkspaceMenu { + private Integer id; + private String name; + private String titleCn; + private String titleEn; + private String description; + private Boolean isActive; + private String icon; + private Integer order; + private String createBy; + private Date createTime; + private Date lastUpdateTime; + private String lastUpdateUser; + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getTitleCn() { + return titleCn; + } + + public void setTitleCn(String titleCn) { + this.titleCn = titleCn; + } + + public String getTitleEn() { + return titleEn; + } + + public void setTitleEn(String titleEn) { + this.titleEn = titleEn; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Boolean getActive() { + return isActive; + } + + public void setActive(Boolean active) { + isActive = active; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(Date lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public String getLastUpdateUser() { + return lastUpdateUser; + } + + public void setLastUpdateUser(String lastUpdateUser) { + this.lastUpdateUser = lastUpdateUser; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuComponentUrl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuComponentUrl.java new file mode 100644 index 000000000..23348c5bf --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuComponentUrl.java @@ -0,0 +1,86 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import java.util.Date; + + +public class DSSWorkspaceMenuComponentUrl { + private int id; + private int menuId; + private int dssApplicationId; + private String url; + private String manulUrl; + private String operationUrl; + private Date updateTime; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getMenuId() { + return menuId; + } + + public void setMenuId(int menuId) { + this.menuId = menuId; + } + + public int getDssApplicationId() { + return dssApplicationId; + } + + public void setDssApplicationId(int dssApplicationId) { + this.dssApplicationId = dssApplicationId; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getManulUrl() { + return manulUrl; + } + + public void setManulUrl(String manulUrl) { + this.manulUrl = manulUrl; + } + + public String getOperationUrl() { + return operationUrl; + } + + public void setOperationUrl(String operationUrl) { + this.operationUrl = operationUrl; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuRole.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuRole.java new file mode 100644 index 000000000..74c51704f --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuRole.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import java.util.Date; + + +public class DSSWorkspaceMenuRole { + private int id; + private int workspaceId; + private int menuId; + private int roleId; + private int priv; + private Date updateTime; + private String updateBy; + + public DSSWorkspaceMenuRole(int workspaceId, int menuId, int roleId, int priv, Date updateTime, String updateBy) { + this.workspaceId = workspaceId; + this.menuId = menuId; + this.roleId = roleId; + this.priv = priv; + this.updateTime = updateTime; + this.updateBy = updateBy; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public int getMenuId() { + return menuId; + } + + public void setMenuId(int menuId) { + this.menuId = menuId; + } + + public int getRoleId() { + return roleId; + } + + public void setRoleId(int roleId) { + this.roleId = roleId; + } + + public int getPriv() { + return priv; + } + + public void setPriv(int priv) { + this.priv = priv; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuRolePriv.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuRolePriv.java new file mode 100644 index 000000000..07d83b999 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceMenuRolePriv.java @@ -0,0 +1,106 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.google.common.base.Objects; + +import java.util.Date; + + +public class DSSWorkspaceMenuRolePriv { + private int id; + private int workspaceId; + private int menuId; + private int roleId; + private int priv; + private Date updateTime; + private String updateBy; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public int getMenuId() { + return menuId; + } + + public void setMenuId(int menuId) { + this.menuId = menuId; + } + + public int getRoleId() { + return roleId; + } + + public void setRoleId(int roleId) { + this.roleId = roleId; + } + + public int getPriv() { + return priv; + } + + public void setPriv(int priv) { + this.priv = priv; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DSSWorkspaceMenuRolePriv that = (DSSWorkspaceMenuRolePriv) o; + return Objects.equal(menuId, that.menuId) && + Objects.equal(roleId, that.roleId); + } + + @Override + public int hashCode() { + return Objects.hashCode(menuId, roleId); + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceRole.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceRole.java new file mode 100644 index 000000000..595c3a974 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceRole.java @@ -0,0 +1,89 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import java.util.Date; + + +public class DSSWorkspaceRole { + private int id; + private int workspaceId; + private String name; + private String frontName; + private Date createTime; + private String description; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFrontName() { + return frontName; + } + + public void setFrontName(String frontName) { + this.frontName = frontName; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + @Override + public String toString() { + return "DSSRole{" + + "id=" + id + + ", workspaceId=" + workspaceId + + ", name='" + name + '\'' + + ", frontName='" + frontName + '\'' + + ", createTime=" + createTime + + ", description='" + description + '\'' + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceUser.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceUser.java new file mode 100644 index 000000000..425296644 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspaceUser.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import java.util.Date; + + +public class DSSWorkspaceUser { + + private Long id; + + private String username; + + private int workspaceId; + + private String creator; + + private Date joinTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Date getJoinTime() { + return joinTime; + } + + public void setJoinTime(Date joinTime) { + this.joinTime = joinTime; + } + + @Override + public String toString() { + return "DSSWorkspaceUser{" + + "id=" + id + + ", username='" + username + '\'' + + ", workspaceId=" + workspaceId + + ", creator='" + creator + '\'' + + ", joinTime=" + joinTime + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/Sidebar.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/Sidebar.java new file mode 100644 index 000000000..8a743fdd6 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/Sidebar.java @@ -0,0 +1,185 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.io.Serializable; +import java.util.Date; + + +@TableName(value = "dss_sidebar") +public class Sidebar implements Serializable { + + private static final long serialVersionUID = -5404077122214161197L; + + @TableId(type = IdType.AUTO) + private Integer id; + + //名称 + private String name; + + //名称(英文) + private String nameEn; + + //标题 + private String title; + + //标题(英文) + private String titleEn; + + //类型: 0-知识库,1-菜单,2-常见问题 + private Integer type; + + //序号 + private Integer orderNum; + + //备注 + private String remark; + + //创建人 + private String createUser; + + //创建时间 + private Date createTime; + + //更新人 + private String updateUser; + + //更新时间 + private Date updateTime; + + + public Sidebar() { + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public Integer getOrderNum() { + return orderNum; + } + + public void setOrderNum(Integer orderNum) { + this.orderNum = orderNum; + } + + public String getRemark() { + return remark; + } + + public void setRemark(String remark) { + this.remark = remark; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getNameEn() { + return nameEn; + } + + public void setNameEn(String nameEn) { + this.nameEn = nameEn; + } + + public String getTitleEn() { + return titleEn; + } + + public void setTitleEn(String titleEn) { + this.titleEn = titleEn; + } + + @Override + public String toString() { + return "Sidebar{" + + "id=" + id + + ", name='" + name + '\'' + + ", nameEn='" + nameEn + '\'' + + ", title='" + title + '\'' + + ", titleEn='" + titleEn + '\'' + + ", type=" + type + + ", orderNum=" + orderNum + + ", remark='" + remark + '\'' + + ", createUser='" + createUser + '\'' + + ", createTime=" + createTime + + ", updateUser='" + updateUser + '\'' + + ", updateTime=" + updateTime + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/SidebarContent.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/SidebarContent.java new file mode 100644 index 000000000..100ba03bb --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/SidebarContent.java @@ -0,0 +1,223 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.io.Serializable; +import java.util.Date; + + +@TableName(value = "dss_sidebar_content") +public class SidebarContent implements Serializable { + + private static final long serialVersionUID = -5404077122214161197L; + + @TableId(type = IdType.AUTO) + private Integer id; + + //侧边栏ID + private Integer sidebarId; + + //名称 + private String name; + + //名称(英文) + private String nameEn; + + //标题 + private String title; + + //标题(英文) + private String titleEn; + + //url + private String url; + + /** + * url类型: 0-内部系统,1-外部系统;默认是内部 + */ + private Integer urlType; + + //icon + private String icon; + + //序号 + private Integer orderNum; + + //备注 + private String remark; + + //创建人 + private String createUser; + + //创建时间 + private Date createTime; + + //更新人 + private String updateUser; + + //更新时间 + private Date updateTime; + + + public SidebarContent() { + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Integer getSidebarId() { + return sidebarId; + } + + public void setSidebarId(Integer sidebarId) { + this.sidebarId = sidebarId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public Integer getUrlType() { + return urlType; + } + + public void setUrlType(Integer urlType) { + this.urlType = urlType; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public Integer getOrderNum() { + return orderNum; + } + + public void setOrderNum(Integer orderNum) { + this.orderNum = orderNum; + } + + public String getRemark() { + return remark; + } + + public void setRemark(String remark) { + this.remark = remark; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getNameEn() { + return nameEn; + } + + public void setNameEn(String nameEn) { + this.nameEn = nameEn; + } + + public String getTitleEn() { + return titleEn; + } + + public void setTitleEn(String titleEn) { + this.titleEn = titleEn; + } + + @Override + public String toString() { + return "SidebarContent{" + + "id=" + id + + ", sidebarId=" + sidebarId + + ", name='" + name + '\'' + + ", nameEn='" + nameEn + '\'' + + ", title='" + title + '\'' + + ", titleEn='" + titleEn + '\'' + + ", url='" + url + '\'' + + ", urlType=" + urlType + + ", icon='" + icon + '\'' + + ", orderNum=" + orderNum + + ", remark='" + remark + '\'' + + ", createUser='" + createUser + '\'' + + ", createTime=" + createTime + + ", updateUser='" + updateUser + '\'' + + ", updateTime=" + updateTime + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/StaffInfo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/StaffInfo.java new file mode 100644 index 000000000..ea422578e --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/StaffInfo.java @@ -0,0 +1,150 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.annotate.JsonProperty; + + +@JsonIgnoreProperties(ignoreUnknown = true) +public class StaffInfo { + @JsonProperty(value = "ID") + private String id; + @JsonProperty(value = "StaffID") + private String staffId; + @JsonProperty(value = "OID") + private String oId; + @JsonProperty(value = "ChineseName") + private String chineseName; + @JsonProperty(value = "EnglishName") + private String englishName; + @JsonProperty(value = "FullName") + private String fullName; + @JsonProperty(value = "OrgName") + private String orgName; + @JsonProperty(value = "OrgFullName") + private String orgFullName; + @JsonProperty(value = "Status") + private String status; + @JsonProperty(value = "PersonGroup") + private String personGroup; + + public StaffInfo(){ + } + + public StaffInfo(String id,String englishName, String orgFullName){ + this.id = id; + this.englishName = englishName; + this.orgFullName = orgFullName; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStaffId() { + return staffId; + } + + public void setStaffId(String staffId) { + this.staffId = staffId; + } + + public String getoId() { + return oId; + } + + public void setoId(String oId) { + this.oId = oId; + } + + public String getChineseName() { + return chineseName; + } + + public void setChineseName(String chineseName) { + this.chineseName = chineseName; + } + + public String getEnglishName() { + return englishName; + } + + public void setEnglishName(String englishName) { + this.englishName = englishName; + } + + public String getFullName() { + return fullName; + } + + public void setFullName(String fullName) { + this.fullName = fullName; + } + + public String getOrgName() { + return orgName; + } + + public void setOrgName(String orgName) { + this.orgName = orgName; + } + + public String getOrgFullName() { + return orgFullName; + } + + public void setOrgFullName(String orgFullName) { + this.orgFullName = orgFullName; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getPersonGroup() { + return personGroup; + } + + public void setPersonGroup(String personGroup) { + this.personGroup = personGroup; + } + + @Override + public String toString() { + return "StaffInfo{" + + "id='" + id + '\'' + + ", staffId='" + staffId + '\'' + + ", oId='" + oId + '\'' + + ", chineseName='" + chineseName + '\'' + + ", englishName='" + englishName + '\'' + + ", fullName='" + fullName + '\'' + + ", orgName='" + orgName + '\'' + + ", orgFullName='" + orgFullName + '\'' + + ", status='" + status + '\'' + + ", personGroup='" + personGroup + '\'' + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/HomepageDemoInstanceVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/HomepageDemoInstanceVo.java new file mode 100644 index 000000000..66096fde2 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/HomepageDemoInstanceVo.java @@ -0,0 +1,93 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.framework.workspace.bean.dto.response; + +@Deprecated +public class HomepageDemoInstanceVo { + + private Long id; + private Long menuId; + private String name; + private String url; + private String title; + private String description; + private String icon; + private Integer order; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getMenuId() { + return menuId; + } + + public void setMenuId(Long menuId) { + this.menuId = menuId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceDepartmentVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceDepartmentVo.java new file mode 100644 index 000000000..fb96389d7 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceDepartmentVo.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.dto.response; + + +public class WorkspaceDepartmentVo { + + private Long id; + + private String name; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceFavoriteVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceFavoriteVo.java new file mode 100644 index 000000000..94c5be6b5 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceFavoriteVo.java @@ -0,0 +1,90 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.dto.response; + +import com.google.common.base.Objects; + + +public class WorkspaceFavoriteVo { + private Long id; + + private Long menuApplicationId; + + private String name; + + private String icon; + + private String title; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Long getMenuApplicationId() { + return menuApplicationId; + } + + public void setMenuApplicationId(Long menuApplicationId) { + this.menuApplicationId = menuApplicationId; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + WorkspaceFavoriteVo that = (WorkspaceFavoriteVo) o; + return Objects.equal(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hashCode(name); + } + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceMenuAppconnVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceMenuAppconnVo.java new file mode 100644 index 000000000..20219fa0e --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceMenuAppconnVo.java @@ -0,0 +1,202 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.framework.workspace.bean.dto.response; + +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSApplicationBean; + +import java.util.List; + + +public class WorkspaceMenuAppconnVo { + private Long id; + private String title; + private String description; + private String labels; + // private String accessButton; +// private String accessButtonUrl; +// private String manualButton; +// private String manualButtonUrl; +// private String projectUrl; + private String name; + private Boolean isActive; + private Boolean accessable; + private String icon; + private Integer order; + // private Map nameAndUrls; + //图片 + private String image; + + private Boolean ifIframe; + private Boolean isExternal; + + private List appInstances; + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } +// +// public String getAccessButton() { +// return accessButton; +// } +// +// public void setAccessButton(String accessButton) { +// this.accessButton = accessButton; +// } +// +// public String getAccessButtonUrl() { +// return accessButtonUrl; +// } +// +// public void setAccessButtonUrl(String accessButtonUrl) { +// this.accessButtonUrl = accessButtonUrl; +// } +// +// public String getManualButton() { +// return manualButton; +// } +// +// public void setManualButton(String manualButton) { +// this.manualButton = manualButton; +// } +// +// public String getManualButtonUrl() { +// return manualButtonUrl; +// } +// +// public void setManualButtonUrl(String manualButtonUrl) { +// this.manualButtonUrl = manualButtonUrl; +// } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } +// +// public String getProjectUrl() { +// return projectUrl; +// } +// +// public void setProjectUrl(String projectUrl) { +// this.projectUrl = projectUrl; +// } + + public Boolean getActive() { + return isActive; + } + + public void setActive(Boolean active) { + isActive = active; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + +// public Map getNameAndUrls() { +// return nameAndUrls; +// } +// +// public void setNameAndUrls(Map nameAndUrls) { +// this.nameAndUrls = nameAndUrls; +// } + + public Boolean getAccessable() { + return accessable; + } + + public void setAccessable(Boolean accessable) { + this.accessable = accessable; + } + + public List getAppInstances() { + return appInstances; + } + + public void setAppInstances(List appInstances) { + this.appInstances = appInstances; + } + + public void addAppInstances(DSSApplicationBean dssApplicationBean) { + this.appInstances.add(dssApplicationBean); + } + + public Boolean getIfIframe() { + return ifIframe; + } + + public void setIfIframe(Boolean ifIframe) { + this.ifIframe = ifIframe; + } + + public Boolean getExternal() { + return isExternal; + } + + public void setExternal(Boolean external) { + isExternal = external; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceMenuVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceMenuVo.java new file mode 100644 index 000000000..57e285aa9 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/dto/response/WorkspaceMenuVo.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.dto.response; + +import java.util.List; + + +public class WorkspaceMenuVo { + private Long id; + private String title; + private Integer order; + private List appconns; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } + + public List getAppconns() { + return appconns; + } + + public void setAppconns(List appInstances) { + this.appconns = appInstances; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/AddWorkspaceRoleRequest.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/AddWorkspaceRoleRequest.java new file mode 100644 index 000000000..c0cee240e --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/AddWorkspaceRoleRequest.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.request; + +import java.io.Serializable; +import java.util.List; + +public class AddWorkspaceRoleRequest implements Serializable { + + private int workspaceId; + private String roleName ; + private List menuIds; + private List componentIds; + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public String getRoleName() { + return roleName; + } + + public void setRoleName(String roleName) { + this.roleName = roleName; + } + + public List getMenuIds() { + return menuIds; + } + + public void setMenuIds(List menuIds) { + this.menuIds = menuIds; + } + + public List getComponentIds() { + return componentIds; + } + + public void setComponentIds(List componentIds) { + this.componentIds = componentIds; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/CreateWorkspaceRequest.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/CreateWorkspaceRequest.java new file mode 100644 index 000000000..5cbafc1d5 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/CreateWorkspaceRequest.java @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.request; + +import java.io.Serializable; + +public class CreateWorkspaceRequest implements Serializable { + + private String workspaceName; + private String department; + private String description; + private String tags; + private String productName; + + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getTags() { + return tags; + } + + public void setTags(String tags) { + this.tags = tags; + } + + public String getProductName() { + return productName; + } + + public void setProductName(String productName) { + this.productName = productName; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/DeleteWorkspaceUserRequest.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/DeleteWorkspaceUserRequest.java new file mode 100644 index 000000000..8fc2a2cf2 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/DeleteWorkspaceUserRequest.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.request; + +import java.io.Serializable; + +public class DeleteWorkspaceUserRequest implements Serializable { + + private int workspaceId; + private String userName; + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String username) { + this.userName = username; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateRoleComponentPrivRequest.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateRoleComponentPrivRequest.java new file mode 100644 index 000000000..8976c2e3e --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateRoleComponentPrivRequest.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.request; + + +import java.io.Serializable; +import java.util.Map; + +public class UpdateRoleComponentPrivRequest implements Serializable { + + public int componentId; + public int workspaceId; + public Map componentPrivs; + + public int getComponentId() { + return componentId; + } + + public void setComponentId(int componentId) { + this.componentId = componentId; + } + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public Map getComponentPrivs() { + return componentPrivs; + } + + public void setComponentPrivs(Map componentPrivs) { + this.componentPrivs = componentPrivs; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateRoleMenuPrivRequest.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateRoleMenuPrivRequest.java new file mode 100644 index 000000000..697b3b5be --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateRoleMenuPrivRequest.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.request; + + +import java.io.Serializable; +import java.util.Map; + +public class UpdateRoleMenuPrivRequest implements Serializable { + + private int menuId; + private int workspaceId; + private Map menuPrivs; + + public int getMenuId() { + return menuId; + } + + public void setMenuId(int menuId) { + this.menuId = menuId; + } + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public Map getMenuPrivs() { + return menuPrivs; + } + + public void setMenuPrivs(Map menuPrivs) { + this.menuPrivs = menuPrivs; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateWorkspaceUserRequest.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateWorkspaceUserRequest.java new file mode 100644 index 000000000..3aff4a07f --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/request/UpdateWorkspaceUserRequest.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.request; + +import java.io.Serializable; +import java.util.List; + +public class UpdateWorkspaceUserRequest implements Serializable { + + private int workspaceId; + private String userName ; + private List roles; + private String userId; + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + + public String getUserId() { + return userId; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/AbstractDSSVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/AbstractDSSVO.java new file mode 100644 index 000000000..a18bdfd4c --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/AbstractDSSVO.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + + +public abstract class AbstractDSSVO { +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSDictionaryRequestVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSDictionaryRequestVO.java new file mode 100644 index 000000000..894cf5f33 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSDictionaryRequestVO.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +public class DSSDictionaryRequestVO { + + private Integer workspaceId; + + private String dicKey; + + private String parentKey; + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public String getDicKey() { + return dicKey; + } + + public void setDicKey(String dicKey) { + this.dicKey = dicKey; + } + + public String getParentKey() { + return parentKey; + } + + public void setParentKey(String parentKey) { + this.parentKey = parentKey; + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceComponentPrivVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceComponentPrivVO.java new file mode 100644 index 000000000..7e217f7fe --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceComponentPrivVO.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.io.Serializable; +import java.util.Map; + + +public class DSSWorkspaceComponentPrivVO extends AbstractDSSVO implements Serializable { + private static final long serialVersionUID=1L; + + private int id; + private String name; + private Map componentPrivs; + + + public DSSWorkspaceComponentPrivVO() { + } + + public DSSWorkspaceComponentPrivVO(int id, String name, Map componentPrivs) { + this.id = id; + this.name = name; + this.componentPrivs = componentPrivs; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map getComponentPrivs() { + return componentPrivs; + } + + public void setComponentPrivs(Map componentPrivs) { + this.componentPrivs = componentPrivs; + } + + @Override + public String toString() { + return "DSSWorkspaceComponentPrivVO{" + + "id=" + id + + ", name='" + name + '\'' + + ", componentPrivs=" + componentPrivs + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceHomePageVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceHomePageVO.java new file mode 100644 index 000000000..35591554e --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceHomePageVO.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + + +public class DSSWorkspaceHomePageVO { + private int workspaceId; + private String roleName; + private String homePageUrl; + + public DSSWorkspaceHomePageVO() { + } + + public DSSWorkspaceHomePageVO(int workspaceId, String roleName, String username, String homePageUrl) { + this.workspaceId = workspaceId; + this.roleName = roleName; + this.homePageUrl = homePageUrl; + } + + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public String getRoleName() { + return roleName; + } + + public void setRoleName(String roleName) { + this.roleName = roleName; + } + + + + public String getHomePageUrl() { + return homePageUrl; + } + + public void setHomePageUrl(String homePageUrl) { + this.homePageUrl = homePageUrl; + } + + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceHomepageSettingVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceHomepageSettingVO.java new file mode 100644 index 000000000..180e0da6d --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceHomepageSettingVO.java @@ -0,0 +1,86 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.util.List; +import java.util.Map; + + +public class DSSWorkspaceHomepageSettingVO { + + public static class RoleHomepage{ + private String roleName; + private String homepageName; + private String homepageUrl; + private int roleId; + private String roleFrontName; + + + public String getRoleName() { + return roleName; + } + + public void setRoleName(String roleName) { + this.roleName = roleName; + } + + public String getHomepageName() { + return homepageName; + } + + public void setHomepageName(String homepageName) { + this.homepageName = homepageName; + } + + public String getHomepageUrl() { + return homepageUrl; + } + + public void setHomepageUrl(String homepageUrl) { + this.homepageUrl = homepageUrl; + } + + + public int getRoleId() { + return roleId; + } + + public void setRoleId(int roleId) { + this.roleId = roleId; + } + + public String getRoleFrontName() { + return roleFrontName; + } + + public void setRoleFrontName(String roleFrontName) { + this.roleFrontName = roleFrontName; + } + } + + private List roleHomepages; + + public List getRoleHomepages() { + return roleHomepages; + } + + public void setRoleHomepages(List roleHomepages) { + this.roleHomepages = roleHomepages; + } + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceMenuPrivVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceMenuPrivVO.java new file mode 100644 index 000000000..ed4c86620 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceMenuPrivVO.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.io.Serializable; +import java.util.Map; + + +public class DSSWorkspaceMenuPrivVO extends AbstractDSSVO implements Serializable { + private static final long serialVersionUID=1L; + private int id; + private String name; + private Map menuPrivs; + + public DSSWorkspaceMenuPrivVO() { + } + + public DSSWorkspaceMenuPrivVO(int id, String name, Map menuPrivs) { + this.id = id; + this.name = name; + this.menuPrivs = menuPrivs; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map getMenuPrivs() { + return menuPrivs; + } + + public void setMenuPrivs(Map menuPrivs) { + this.menuPrivs = menuPrivs; + } + + @Override + public String toString() { + return "DSSWorkspaceMenuPrivVO{" + + "id=" + id + + ", name='" + name + '\'' + + ", menuPrivs=" + menuPrivs + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceOverviewVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceOverviewVO.java new file mode 100644 index 000000000..885b20493 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceOverviewVO.java @@ -0,0 +1,119 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.util.List; +import java.util.Map; + + +public class DSSWorkspaceOverviewVO { + private String title; + + + private String description; + + private String dssDescription; + + + + + private List videos; + + + Map> demos; + + public static class OverviewInfo{ + private int id; + private String title; + private String url; + + public OverviewInfo(){ + + } + + public OverviewInfo(String title, String url){ + this.title = title; + this.url = url; + } + + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDssDescription() { + return dssDescription; + } + + public void setDssDescription(String dssDescription) { + this.dssDescription = dssDescription; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + + public Map> getDemos() { + return demos; + } + + public void setDemos(Map> demos) { + this.demos = demos; + } + + public List getVideos() { + return videos; + } + + public void setVideos(List videos) { + this.videos = videos; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspacePrivVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspacePrivVO.java new file mode 100644 index 000000000..5b4a15dc1 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspacePrivVO.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.io.Serializable; +import java.util.List; + + +public class DSSWorkspacePrivVO implements Serializable { + private static final long serialVersionUID = 1L; + private int workspaceId; + private List menuPrivVOS; + private List componentPrivVOS; + private List roleVOS; + + public List getRoleVOS() { + return roleVOS; + } + + public void setRoleVOS(List roleVOS) { + this.roleVOS = roleVOS; + } + + + public DSSWorkspacePrivVO() { + } + + public DSSWorkspacePrivVO(int workspaceId, List menuPrivVOS, List componentPrivVOS, List roleVOS) { + this.workspaceId = workspaceId; + this.menuPrivVOS = menuPrivVOS; + this.componentPrivVOS = componentPrivVOS; + this.roleVOS = roleVOS; + } + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + public List getMenuPrivVOS() { + return menuPrivVOS; + } + + public void setMenuPrivVOS(List menuPrivVOS) { + this.menuPrivVOS = menuPrivVOS; + } + + public List getComponentPrivVOS() { + return componentPrivVOS; + } + + public void setComponentPrivVOS(List componentPrivVOS) { + this.componentPrivVOS = componentPrivVOS; + } + + @Override + public String toString() { + return "DSSWorkspacePrivVO{" + + "workspaceId=" + workspaceId + + ", menuPrivVOS=" + menuPrivVOS + + ", componentPrivVOS=" + componentPrivVOS + + ", roleVOS=" + roleVOS + + '}'; + } + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceRoleVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceRoleVO.java new file mode 100644 index 000000000..3e0d681df --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceRoleVO.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.io.Serializable; + + +public class DSSWorkspaceRoleVO implements Serializable { + private static final long serialVersionUID=1L; + + private int roleId; + private String roleName; + private String roleFrontName; + + + public int getRoleId() { + return roleId; + } + + public void setRoleId(int roleId) { + this.roleId = roleId; + } + + public String getRoleName() { + return roleName; + } + + public void setRoleName(String roleName) { + this.roleName = roleName; + } + + public String getRoleFrontName() { + return roleFrontName; + } + + public void setRoleFrontName(String roleFrontName) { + this.roleFrontName = roleFrontName; + } + + @Override + public String toString() { + return "DSSWorkspaceRoleVO{" + + "roleId=" + roleId + + ", roleName='" + roleName + '\'' + + ", roleFrontName='" + roleFrontName + '\'' + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUserVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUserVO.java new file mode 100644 index 000000000..a3aa5775f --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUserVO.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + + +public class DSSWorkspaceUserVO extends AbstractDSSVO{ + private int id; + private String name; + private List roles; + private String department; + private String office; + private String creator; + private Date joinTime; + + public DSSWorkspaceUserVO() { + } + + + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } + + public String getOffice() { + return office; + } + + public void setOffice(String office) { + this.office = office; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Date getJoinTime() { + return joinTime; + } + + public void setJoinTime(Date joinTime) { + this.joinTime = joinTime; + } + + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersVo.java new file mode 100644 index 000000000..82a562dc0 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersVo.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.util.List; + +public class DSSWorkspaceUsersVo { + + private List editUsers; + + private List accessUsers; + + private List releaseUsers; + + public List getAccessUsers() { + return accessUsers; + } + + public List getEditUsers() { + return editUsers; + } + + public List getReleaseUsers() { + return releaseUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public void setReleaseUsers(List releaseUsers) { + this.releaseUsers = releaseUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceVO.java new file mode 100644 index 000000000..c5f90b30a --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceVO.java @@ -0,0 +1,89 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + + +public class DSSWorkspaceVO extends AbstractDSSVO{ + + + private int id; + private String name; + private String tags; + private String department; + private String description; + private String product; + + + public DSSWorkspaceVO() { + } + + public DSSWorkspaceVO(int id, String name){ + this.id = id; + this.name = name; + } + + + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTags() { + return tags; + } + + public void setTags(String tags) { + this.tags = tags; + } + + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentVO.java new file mode 100644 index 000000000..3e5a70582 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentVO.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + + +public class DepartmentVO { + private int id; + private String name; + private String frontName; + + public DepartmentVO(){ + + } + + public DepartmentVO(int id, String name, String frontName) { + this.id = id; + this.name = name; + this.frontName = frontName; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFrontName() { + return frontName; + } + + public void setFrontName(String frontName) { + this.frontName = frontName; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/SidebarContentVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/SidebarContentVO.java new file mode 100644 index 000000000..694b93a42 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/SidebarContentVO.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +public class SidebarContentVO { + + //名称 + private String name; + //标题 + private String title; + + private String url; + /** + * url类型: 0-内部系统,1-外部系统;默认是内部 + */ + private Integer urlType; + /** + * icon是表示Content的图标,如果为空就是没有 + */ + private String icon; + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public Integer getUrlType() { + return urlType; + } + + public void setUrlType(Integer urlType) { + this.urlType = urlType; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/SidebarVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/SidebarVO.java new file mode 100644 index 000000000..a9dfb8f35 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/SidebarVO.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.util.List; + +public class SidebarVO { + + + private String name; + + + private String title; + + private Integer type; + + private List contents; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public List getContents() { + return contents; + } + + public void setContents(List contents) { + this.contents = contents; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/StaffInfoVO.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/StaffInfoVO.java new file mode 100644 index 000000000..caeb7a8e6 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/StaffInfoVO.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + + +public class StaffInfoVO { + private String id; + private String username; + private String department; + private String office; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } + + public String getOffice() { + return office; + } + + public void setOffice(String office) { + this.office = office; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/constant/ApplicationConf.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/constant/ApplicationConf.java new file mode 100644 index 000000000..50feb60e8 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/constant/ApplicationConf.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.constant; + + +import org.apache.linkis.common.conf.CommonVars; + + +public class ApplicationConf { + + public static final CommonVars FAQ = CommonVars.apply("wds.linkis.application.dws.params","http://127.0.0.1:8088/wiki/scriptis/manual/feature_overview_cn.html"); + public static final CommonVars SCHEDULIS_URL = + CommonVars.apply("wds.linkis.schedulis.url", "http://127.0.0.1:8088"); + + public static final CommonVars HOMEPAGE_MODULE_NAME = + CommonVars.apply("wds.linkis.special.homepage.module.name", "apiServices"); + + public static final CommonVars HOMEPAGE_URL = + CommonVars.apply("wds.linkis.special.homepage.module.url", "/newHome?workspaceId="); + + public static final CommonVars DSS_ENV_PROD_LABEL = + CommonVars.apply("wds.dss.env.prod.label", "PROD"); + + public static final String SCHEDULER_APP_CONN_NAME = CommonVars.apply("wds.dss.appconn.scheduler.name", "schedulis").getValue(); + + public static final String ESB_APPID = CommonVars.apply("wds.dss.esb.appid", "").getValue(); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSComponentRoleMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSComponentRoleMapper.java new file mode 100644 index 000000000..a52fb355b --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSComponentRoleMapper.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSComponentRole; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceComponentPriv; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceHomepage; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +public interface DSSComponentRoleMapper extends BaseMapper { + + + public void insertBatch(@Param("list") List list); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSDictionaryMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSDictionaryMapper.java new file mode 100644 index 000000000..e120caec7 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSDictionaryMapper.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSDictionary; +import org.apache.ibatis.annotations.Mapper; + + +@Mapper +public interface DSSDictionaryMapper extends BaseMapper { + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSMenuRoleMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSMenuRoleMapper.java new file mode 100644 index 000000000..d7f215aae --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSMenuRoleMapper.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceMenuRole; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSMenuRole; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +public interface DSSMenuRoleMapper extends BaseMapper { + + public void insertBatch(@Param("list") List menuRoles); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceHomepageMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceHomepageMapper.java new file mode 100644 index 000000000..c183ad978 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceHomepageMapper.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceHomepage; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceMenuRole; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +public interface DSSWorkspaceHomepageMapper extends BaseMapper { + + public void insertBatch(@Param("list") List homepageList); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceInfoMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceInfoMapper.java new file mode 100644 index 000000000..4c197851f --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceInfoMapper.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace; +import org.apache.ibatis.annotations.*; + + +@Mapper +public interface DSSWorkspaceInfoMapper { + + @Select("select name from dss_workspace where id = #{workspaceId} ") + String getWorkspaceNameById(@Param("workspaceId") int workspaceId); + + @Select("select id from dss_workspace where name = #{workspaceName}") + Integer getWorkspaceIdByName(@Param("workspaceName") String workspaceName); + + @Select("select * from dss_workspace where id = #{workspaceId}") + @Results({ + @Result(property = "createBy", column = "create_by"), + @Result(property = "createTime", column = "create_time"), + @Result(property = "lastUpdateTime", column = "last_update_time"), + @Result(property = "lastUpdateUser", column = "last_update_user"), + }) + DSSWorkspace getWorkspace(@Param("workspaceId") int workspaceId); +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMapper.java new file mode 100644 index 000000000..8c038deb6 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMapper.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import com.webank.wedatasphere.dss.framework.workspace.bean.*; +import org.apache.ibatis.annotations.*; + +import java.util.List; + + +@Mapper +public interface DSSWorkspaceMapper { + + void createWorkSpace(DSSWorkspace dssWorkspace); + + List getWorkspaces(String username); + + List getUserMenuAppConnId(@Param("username")String username, @Param("workspaceId")Long workspaceId); + List getMenuId(int roleId, String workspaceId); + + List getDSSWorkspaceMenuPriv(String workspaceId); + + @Select("select -1 as workspaceId, id as menu_id, 1 as role_id, 1 as priv from dss_workspace_menu") + @Results({ + @Result(property = "workspaceId",column = "workspace_id"), + @Result(property = "menuId",column = "menu_id"), + @Result(property = "roleId",column = "role_id"), + @Result(property = "priv",column = "priv") + }) + List getDefaultWorkspaceMenuPriv(); + + @Insert({ + "" + }) + void setDefaultComponentRoles(@Param("privs") List dssWorkspaceComponentPrivs); + + @Select("select count(1) from dss_workspace_user_favorites_appconn where menu_appconn_id=#{menuAppId} and workspace_id=#{workspaceId}" + + " and username=#{userName} and type=#{type}") + int getByMenuAppIdAndUser(@Param("menuAppId") Long menuAppId, @Param("workspaceId") Long workspaceId, + @Param("userName") String userName, @Param("type") String type); + + @Select("select id from dss_workspace_menu_appconn where title_en=#{appName}") + Long getMenuAppIdByName(@Param("appName") String appName); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMenuMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMenuMapper.java new file mode 100644 index 000000000..f3e88a5cd --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMenuMapper.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceComponentRolePriv; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceHomepageSetting; +import org.apache.ibatis.annotations.*; + +import java.util.List; + + +@Mapper +public interface DSSWorkspaceMenuMapper { + + @Select("select * from dss_workspace_appconn_role where workspace_id = #{workspaceId}") + @Results({ + @Result(property = "workspaceId", column = "workspace_id"), + @Result(property = "componentId", column = "appconn_id"), + @Result(property = "roleId", column = "role_id"), + @Result(property = "lastUpdateTime", column = "last_update_time"), + @Result(property = "updateBy", column = "update_by") + }) + List getComponentRolePriv(@Param("workspaceId") int workspaceId); + + @Select("select * from dss_workspace_homepage where workspace_id = #{workspaceId}") + @Results({ + @Result(property = "workspaceId", column = "workspace_id"), + @Result(property = "homepageUrl", column = "homepage_url"), + @Result(property = "roleId", column = "role_id"), + @Result(property = "updateTime", column = "update_time") + }) + List getWorkspaceHomepageSettings(@Param("workspaceId") int workspaceId); + + @Select("select priv from dss_workspace_appconn_role where workspace_id = #{workspaceId} " + + "and role_id = #{roleId} and " + + "appconn_id = #{applicationId}") + Integer getOneCompoentRolePriv(@Param("workspaceId") int workspaceId, + @Param("roleId") int roleId, @Param("applicationId") int applicationId); + + /** + * 如果在dss_workspace_appconn_role中是有workspace_id=-1的情况, + * 默认是-1的时候,就要全部拿出来 + * @return + */ + @Select("select * from dss_workspace_appconn_role where workspace_id = -1") + @Results({ + @Result(property = "workspaceId", column = "workspace_id"), + @Result(property = "componentId", column = "appconn_id"), + @Result(property = "roleId", column = "role_id"), + @Result(property = "lastUpdateTime", column = "last_update_time"), + @Result(property = "updateBy", column = "update_by") + }) + List getDefaultComponentRolePriv(); + + + @Select("select -1 as workspace_id,appconn_id as appconn_id, 1 as role_id, 1 as priv, last_update_time,last_update_user as update_by from dss_workspace_menu_appconn") + @Results({ + @Result(property = "workspaceId", column = "workspace_id"), + @Result(property = "componentId", column = "appconn_id"), + @Result(property = "roleId", column = "role_id"), + @Result(property = "priv", column = "priv"), + @Result(property = "lastUpdateTime", column = "last_update_time"), + @Result(property = "updateBy", column = "update_by") + }) + List getDefaultComponentRolePriv01(); +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspacePrivMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspacePrivMapper.java new file mode 100644 index 000000000..f46f41b9a --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspacePrivMapper.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + + +@Mapper +public interface DSSWorkspacePrivMapper { + + @Update("update dss_workspace_menu_role set priv = #{priv}, update_time = now() " + + "where workspace_id = #{workspaceId} and menu_id = #{menuId} and role_id = #{roleId}") + void updateRoleMenuPriv(@Param("workspaceId") int workspaceId, @Param("menuId") int menuId, + @Param("roleId") int roleId, @Param("priv") int priv); + + @Select("select count(*) from dss_workspace_menu_role where workspace_id = #{workspaceId} and menu_id = #{menuId} and role_id = #{roleId}") + int queryCntOfMenuRolePriv(@Param("workspaceId") int workspaceId, @Param("menuId") int menuId, @Param("roleId") int roleId); + + @Select("insert into dss_workspace_menu_role (`workspace_id`, `menu_id`, `role_id`, `priv`, `update_time`, `updateby`) " + + "values(#{workspaceId}, #{menuId}, #{roleId}, #{priv}, now(), #{username})") + void insertMenuRolePriv(@Param("workspaceId") int workspaceId, @Param("menuId") int menuId, @Param("roleId") int roleId, + @Param("priv") int priv, + @Param("username") String username); + + @Update("update dss_workspace_appconn_role set priv = #{priv} , update_time = now()" + + "where workspace_id = #{workspaceId} and appconn_id = #{componentId} and role_id = #{roleId}") + void updateRoleComponentPriv(@Param("workspaceId") int workspaceId, @Param("componentId") int componentId, + @Param("roleId") int roleId, @Param("priv") int priv); + + @Select("select id from dss_workspace_role where workspace_id = #{workspaceId} and name = #{key}") + Integer getRoleId(@Param("workspaceId") int workspaceId, @Param("key") String key); + + @Select("select count(*) from dss_workspace_appconn_role where workspace_id = #{workspaceId} and appconn_id = #{componentId} and role_id = #{roleId}") + int queryCntOfRCP(@Param("workspaceId") int workspaceId, @Param("componentId") int componentId, @Param("roleId") int roleId); + + @Select("insert into dss_workspace_appconn_role (`workspace_id`, `appconn_id`, `role_id`, `priv`, `update_time`, `updateby`) " + + "values(#{workspaceId}, #{componentId}, #{roleId}, #{priv}, now(), #{username})") + void insertRolComponentPriv(@Param("workspaceId") int workspaceId, @Param("componentId") int componentId, @Param("roleId") int roleId, + @Param("priv") int priv, + @Param("username") String username); +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceRoleMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceRoleMapper.java new file mode 100644 index 000000000..c757fa883 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceRoleMapper.java @@ -0,0 +1,111 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import com.webank.wedatasphere.dss.framework.workspace.bean.*; +import org.apache.ibatis.annotations.*; + +import java.util.List; + + +@Mapper +public interface DSSWorkspaceRoleMapper { + + @Select("select * from dss_workspace_role") + @Results({ + @Result(property = "workspaceId", column = "workspace_id"), + @Result(property = "frontName", column = "front_name"), + @Result(property = "createTime", column = "update_time") + }) + List getRoles(); + + + @Select("select a.`name` from dss_workspace_role a join dss_workspace_user_role b on a.id = b.role_id" + + " and b.`username` = #{username} and b.workspace_id = #{workspaceId}") + List getAllRoles(@Param("username") String username, @Param("workspaceId") int workspaceId); + + + @Select("select * from dss_workspace_menu") + @Results({ + @Result(property = "titleCn", column = "title_cn"), + @Result(property = "titleEn", column = "title_En") + }) + List getWorkspaceMenus(); + + + @Select("select a.id id, a.appconn_name name,b.url url, b.homepage_uri homepage_uri, b.label label from dss_appconn a join dss_appconn_instance b on a.id=b.appconn_id") + @Results(value = { + @Result(property = "id", column = "id"), + @Result(property = "name", column = "name"), + @Result(property = "url", column = "url"), + @Result(property = "homepageUri", column = "homepage_uri"), + @Result(property = "label", column = "label") + }) + List getDSSAppConns(); + + @Insert("insert into dss_workspace_role(workspace_id, name, front_name, update_time) " + + "values(#{dssRole.workspaceId}, #{dssRole.name}, #{dssRole.frontName}, #{dssRole.createTime})") + @Options(useGeneratedKeys = true, keyProperty = "dssRole.id", keyColumn = "id") + int addNewRole(@Param("dssRole") DSSWorkspaceRole dssRole); + + + @Insert({ + "" + }) + void updateRoleMenu(@Param("roleId") int roleId, @Param("workspaceId") int workspaceId, + @Param("menuIds") List menuIds, @Param("username") String username, + @Param("priv") Integer priv); + + + + @Insert({ + "" + }) + void updateRoleComponent(@Param("roleId") int roleId, @Param("workspaceId") int workspaceId, + @Param("componentIds") List componentIds, @Param("username") String username, + @Param("priv") Integer priv); + + + @Select("select distinct workspace_id from dss_workspace_user_role where username = #{username}") + List getWorkspaceIds(@Param("username") String username); + + @Select("select workspace_id from dss_workspace where `name` = #{defaultWorkspaceName}") + Integer getDefaultWorkspaceId(@Param("defaultWorkspaceName") String defaultWorkspaceName); + + @Select("select id from dss_workspace_role where workspace_id = #{workspaceId} and name = #{apiUser}") + int getRoleId(@Param("apiUser") String apiUser, @Param("workspaceId") int workspaceId); + + @Select("Select count(*) from dss_workspace_appconn_role where workspace_id = #{workspaceId} and role_id = #{roleId} and appconn_id = #{componentId}") + int getCount(@Param("workspaceId") Integer workspaceId, @Param("componentId") int componentId, @Param("roleId") int roleId); + + @Select("select priv from dss_workspace_appconn_role where workspace_id = #{workspaceId} and role_id = #{roleId} and " + + "appconn_id = #{componentId}") + Integer getPriv(@Param("workspaceId") Integer workspaceId, @Param("roleId") int roleId, @Param("componentId") int componentId); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceUserMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceUserMapper.java new file mode 100644 index 000000000..17e701a22 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceUserMapper.java @@ -0,0 +1,78 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceUser; +import org.apache.ibatis.annotations.*; + +import java.util.List; + + +@Mapper +public interface DSSWorkspaceUserMapper { + + Long getUserID(String userName); + + String getUserName(Long userID); + + @Insert("insert into dss_workspace_user_role(workspace_id, username, role_id, create_time, created_by,user_id)" + + "values(#{workspaceId}, #{username}, #{roleId}, now(), #{createdBy}, #{userId})") + void setUserRoleInWorkspace(@Param("workspaceId") int workspaceId, @Param("roleId") int roleId, + @Param("username") String username, @Param("createdBy") String createdBy, @Param("userId") Long userId); + + @Select("select role_id from dss_workspace_user_role where workspace_id = #{workspaceId} and username = #{username}") + List getRoleInWorkspace(@Param("workspaceId") int workspaceId, @Param("username") String username); + + + @Delete("delete from dss_workspace_user_role where username = #{username} and workspace_id = #{workspaceId}") + void removeAllRolesForUser(@Param("username") String username, @Param("workspaceId") int workspaceId); + + + @Select("select distinct workspace_id from dss_workspace_user_role where username = #{username}") + List getWorkspaceIds(@Param("username") String username); + + @Select("select homepage_url from dss_workspace_homepage where workspace_id = #{workspaceId} and role_id = #{roleId}") + String getHomepageUrl(@Param("workspaceId") int workspaceId, @Param("roleId") int roleId); + + @Select("select distinct username from dss_workspace_user_role where workspace_id = #{workspaceId}") + List getAllWorkspaceUsers(@Param("workspaceId") int workspaceId); + + @Select("select is_admin from dss_user where username = #{userName}") + Integer isAdmin(@Param("userName") String userName); + + @Select({ + "" + }) + List getWorkspaceUsers(@Param("workspaceId") String workspaceId,@Param("username") String username); + + @Select("select distinct created_by as creator, username as username, create_time as joinTime,workspace_id as workspaceId " + + " from dss_workspace_user_role where role_id = #{roleId} and workspace_id = #{workspaceId}") + List getWorkspaceUsersByRole(@Param("workspaceId") int workspaceId, @Param("roleId") int roleId); + + List getWorkspaceEditUsers(int workspaceId); + + List getWorkspaceReleaseUsers(int workspaceId); + + @Select("select count(1) from dss_workspace_user_role where workspace_id = #{workspaceId} and username = #{username}") + Long getCountByUsername(@Param("username") String username, @Param("workspaceId") int workspaceId); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/SidebarContentMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/SidebarContentMapper.java new file mode 100644 index 000000000..2043bf493 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/SidebarContentMapper.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.workspace.bean.SidebarContent; +import org.apache.ibatis.annotations.Mapper; + + +@Mapper +public interface SidebarContentMapper extends BaseMapper { +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/SidebarMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/SidebarMapper.java new file mode 100644 index 000000000..7533055d5 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/SidebarMapper.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.framework.workspace.bean.Sidebar; +import org.apache.ibatis.annotations.Mapper; + + +@Mapper +public interface SidebarMapper extends BaseMapper { + + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/WorkspaceMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/WorkspaceMapper.java new file mode 100644 index 000000000..8bd2844cc --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/WorkspaceMapper.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.dao; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSFavorite; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.*; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +public interface WorkspaceMapper { + + List getWorkspaces(); + + /** + * 获取空间名称 + * @param name + * @return + */ + List findByWorkspaceName(@Param("name") String name); + + void addWorkSpace(DSSWorkspace dssWorkspace); + + DSSWorkspace getWorkspaceById(@Param("workspaceId") Long workspaceId); + + List getManagementMenuCn(); + List getManagementMenuEn(); + + List getAppConnMenuCn(); + List getAppConnMenuEn(); + + List getMenuAppInstancesCn(Long id); + List getMenuAppInstancesEn(Long id); + + List getWorkspaceFavoritesCn(@Param("username") String username, @Param("workspaceId") Long workspaceId,@Param("type") String type); + + List getWorkspaceFavoritesEn(@Param("username") String username, @Param("workspaceId") Long workspaceId,@Param("type") String type); + + void addFavorite(DSSFavorite dssFavorite); + + void deleteFavorite(@Param("username") String username,@Param("applicationId") Long applicationId, @Param("workspaceId") Long workspaceId,@Param("type") String type); + + String getDepartName(@Param("id") Long id); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSComponentRoleMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSComponentRoleMapper.xml new file mode 100644 index 000000000..2eda66f6f --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSComponentRoleMapper.xml @@ -0,0 +1,31 @@ + + + + + + + + insert into dss_workspace_appconn_role (workspace_id, appconn_id, role_id, + priv, update_time, updateby) + values + + (#{item.workspaceId},#{item.componentId},#{item.roleId}, + #{item.priv},#{item.updateTime},#{item.updateBy}) + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSMenuRoleMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSMenuRoleMapper.xml new file mode 100644 index 000000000..dfa15e0ed --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSMenuRoleMapper.xml @@ -0,0 +1,31 @@ + + + + + + + + insert into dss_workspace_menu_role (workspace_id,menu_id,role_id,priv, update_time,updateby) + values + + ( #{item.workspaceId},#{item.menuId},#{item.roleId}, + #{item.priv},#{item.updateTime},#{item.updateBy}) + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceHomepageMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceHomepageMapper.xml new file mode 100644 index 000000000..6638c62df --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceHomepageMapper.xml @@ -0,0 +1,29 @@ + + + + + + + + insert into dss_workspace_homepage (workspace_id, role_id, homepage_url, update_time) + values + + #{homepage.workspaceId}, #{homepage.roleId}, #{homepage.homepageUrl}, #{homepage.updateTime} + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceMapper.xml new file mode 100644 index 000000000..29df8eb46 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceMapper.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id,`name`,`label`,`description`,`create_by`,`create_time`,`department`,`product`,`source`,`last_update_time`,`last_update_user`,`workspace_type` + + + + INSERT INTO dss_workspace() + VALUES + (#{id},#{name},#{label},#{description},#{createBy},#{createTime},#{department},#{product},#{source},#{lastUpdateTime},#{lastUpdateUser},#{workspaceType}) + + + + id,`workspace_id`,`username`,`join_time`,`created_by` + + + + + + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceUserMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceUserMapper.xml new file mode 100644 index 000000000..0a389bb96 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceUserMapper.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/WorkspaceMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/WorkspaceMapper.xml new file mode 100644 index 000000000..269cadd72 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/WorkspaceMapper.xml @@ -0,0 +1,200 @@ + + + + + + + + id,`name`,`label`,`description`,`department`,`product`,`source`,`create_by`,`create_time`,`last_update_user`,`last_update_time` + + + + m.`id`,m.`title_cn` AS `title`, m.`desc_cn` AS `description`,m.`labels_cn` AS `labels`,m.image, + m.`access_button_cn` AS `access_button`,m.`manual_button_cn` AS `manualButton`,m.`is_active`, + m.`manual_button_url`,m.`icon`,m.`order`,app.`homepage_url` AS `access_button_url`,app.project_url, app.`name` + + + m.`id`,m.`title_en` AS `title`, m.`desc_en` AS `description`,m.`labels_en` AS `labels`,m.image, + m.`access_button_en` AS `access_button`,m.`manual_button_en` AS `manualButton`,m.`is_active`, + m.`manual_button_url`,m.`icon`,m.`order`,app.`homepage_url` AS `access_button_url`,app.project_url, app.`name` + + + + id,`username`,`workspace_id`,`menu_appconn_id`,`order`,`create_by`,`create_time`,`last_update_user`,`last_update_time`,`type` + + + + + + + + + + + + + + INSERT INTO dss_workspace () + VALUES + (#{id},#{name},#{label},#{description},#{department},#{product},#{source},#{createBy},#{createTime},#{lastUpdateUser},#{lastUpdateTime}) + + + + + + + + + + + + + + + + + + + + INSERT INTO dss_workspace_user_favorites_appconn () + VALUES + (#{id},#{username},#{workspaceId},#{menuAppConnId},#{order},#{createBy},#{createTime},#{lastUpdateUser},#{lastUpdateTime},#{type}) + + + + DELETE + FROM + dss_workspace_user_favorites_appconn + WHERE + menu_appconn_id = #{applicationId} + AND username = #{username} + AND workspace_id = #{workspaceId} + AND type = #{type} + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/exception/DSSWorkspaceDuplicateNameException.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/exception/DSSWorkspaceDuplicateNameException.java new file mode 100644 index 000000000..87e5b56da --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/exception/DSSWorkspaceDuplicateNameException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class DSSWorkspaceDuplicateNameException extends ErrorException { + + public DSSWorkspaceDuplicateNameException(int errCode, String desc) { + super(errCode, desc); + } + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/exception/DSSWorkspaceLoginFailException.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/exception/DSSWorkspaceLoginFailException.java new file mode 100644 index 000000000..ebc961d33 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/exception/DSSWorkspaceLoginFailException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class DSSWorkspaceLoginFailException extends ErrorException { + public DSSWorkspaceLoginFailException(int errCode, String desc) { + super(errCode, desc); + } + + public DSSWorkspaceLoginFailException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSDictionaryRestful.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSDictionaryRestful.java new file mode 100644 index 000000000..8bba9f7e0 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSDictionaryRestful.java @@ -0,0 +1,123 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.restful; + +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSDictionary; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSDictionaryRequestVO; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSDictionaryService; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import java.util.List; +import java.util.Map; + + +@RequestMapping(path = "/dss/framework/workspace", produces = {"application/json"}) +@RestController +public class DSSDictionaryRestful { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSDictionaryRestful.class); + + @Autowired + private DSSDictionaryService dictionaryService; + + /** + * 数据字典 - 根据key获取 + * @param request + * @param dictionaryRequestVO + * @return + */ + @RequestMapping(path ="getDicList", method = RequestMethod.POST) + public Message getDevFlowList(HttpServletRequest request, @RequestBody DSSDictionaryRequestVO dictionaryRequestVO){ + try{ + if(dictionaryRequestVO.getWorkspaceId()==null){ + return Message.error("workspaceId(空间id)不能为空"); + } + if(StringUtils.isEmpty(dictionaryRequestVO.getParentKey())&&StringUtils.isEmpty(dictionaryRequestVO.getDicKey())){ + return Message.error("获取的parentKey和dicKey不能同时为空"); + } + List dictionaries = dictionaryService.getListByParam(dictionaryRequestVO); + boolean isEnglish = "en".equals(request.getHeader("Content-language")); + if(CollectionUtils.isNotEmpty(dictionaries) && isEnglish){ + dictionaries.stream().forEach(e->international(e)); + } + return Message.ok("获取数据字典成功").data("list", dictionaries); + }catch(Exception e){ + LOGGER.error("Fail to get getDevFlowListError for user {} in workspace {}",dictionaryRequestVO.getWorkspaceId(), e); + return Message.error("获取数据字典失败:"+e.getMessage()); + } + } + + @RequestMapping(path ="getDicSecondList", method = RequestMethod.POST) + public Message getDicSecondList(HttpServletRequest request, @RequestBody DSSDictionaryRequestVO dictionaryRequestVO){ + try{ + if(dictionaryRequestVO.getWorkspaceId()==null){ + return Message.error("workspaceId(空间id)不能为空"); + } + if(StringUtils.isEmpty(dictionaryRequestVO.getParentKey())&&StringUtils.isEmpty(dictionaryRequestVO.getDicKey())){ + return Message.error("获取的parentKey和dicKey不能同时为空"); + } + Map map = dictionaryService.getDicSecondList(dictionaryRequestVO); + boolean isEnglish = "en".equals(request.getHeader("Content-language")); + if(map!=null && map.size() > 0 && isEnglish){ + List dictionaries = (List) map.get("list"); + if(CollectionUtils.isNotEmpty(dictionaries)){ + dictionaries.stream().forEach(e->international(e)); + } + Map> mapList = ( Map>) map.get("mapList"); + if(mapList!=null && mapList.size() > 0){ + for(String key : mapList.keySet()){ + mapList.get(key).stream().forEach(e->international(e)); + } + } + } + return Message.ok("获取数据字典成功").data("list", map); + }catch(Exception e){ + LOGGER.error("Fail to get getDicSecondListError for user {} in workspace {}",dictionaryRequestVO.getWorkspaceId(), e); + return Message.ok("获取数据字典失败:"+e.getMessage()); + } + } + + + //国际化,由于前端只是使用dicName,所以在英文的时候直接将dicNameEn赋值给dicName即可 + public void international(DSSDictionary dssDictionary){ + if(dssDictionary==null){ + return; + } + if(StringUtils.isNotBlank(dssDictionary.getDicNameEn())){ + dssDictionary.setDicName(dssDictionary.getDicNameEn()); + dssDictionary.setDicNameEn(null); + } + if(StringUtils.isNotBlank(dssDictionary.getDicValueEn())){ + dssDictionary.setDicValue(dssDictionary.getDicValueEn()); + dssDictionary.setDicValueEn(null); + } + if(StringUtils.isNotBlank(dssDictionary.getTitleEn())){ + dssDictionary.setTitle(dssDictionary.getTitleEn()); + dssDictionary.setTitleEn(null); + } + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSSideInfoRestful.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSSideInfoRestful.java new file mode 100644 index 000000000..da58c2f5c --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSSideInfoRestful.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.restful; + +import com.webank.wedatasphere.dss.framework.workspace.service.DSSSideInfoService; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; + + +@RequestMapping(path = "/dss/framework/workspace", produces = {"application/json"}) +@RestController +public class DSSSideInfoRestful { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSSideInfoRestful.class); + + @Autowired + private DSSSideInfoService dssSideInfoService; + + @RequestMapping(path ="getSideInfos", method = RequestMethod.GET) + public Message getSideInfos(HttpServletRequest request, @RequestParam(required = false, name = "workspaceID") Long workspaceId){ + String username = SecurityFilter.getLoginUsername(request); + try{ + boolean isEnglish = "en".equals(request.getHeader("Content-language")); + return Message.ok("获取侧边栏成功").data("presentations", dssSideInfoService.getSidebarVOList(username, workspaceId,isEnglish)); + }catch(Exception e){ + LOGGER.info("Fail to get sideinfos for user {} in workspace {}",username, workspaceId, e); + return Message.error("获取侧边栏失败"); + } + } + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspacePrivRestful.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspacePrivRestful.java new file mode 100644 index 000000000..eb98251c7 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspacePrivRestful.java @@ -0,0 +1,107 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.restful; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.request.UpdateRoleComponentPrivRequest; +import com.webank.wedatasphere.dss.framework.workspace.bean.request.UpdateRoleMenuPrivRequest; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceHomepageSettingVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspacePrivVO; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspacePrivService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; +import org.apache.commons.math3.util.Pair; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant.WORKSPACE_ID_STR; + + +@RequestMapping(path = "/dss/framework/workspace", produces = {"application/json"}) +@RestController +public class DSSWorkspacePrivRestful { + + @Autowired + DSSWorkspaceService dssWorkspaceService; + @Autowired + DSSWorkspacePrivService dssWorkspacePrivService; + @Autowired + WorkspaceDBHelper workspaceDBHelper; + + @RequestMapping(path ="getWorkspaceMenuPrivs", method = RequestMethod.GET) + public Message getWorkspaceMenuPrivs(HttpServletRequest request, @RequestParam(WORKSPACE_ID_STR) String workspaceId){ + //todo 返回工作空间中角色对菜单的访问权限 + DSSWorkspacePrivVO workspaceMenuPrivs = dssWorkspaceService.getWorkspaceMenuPrivs(workspaceId); + return Message.ok().data("workspaceMenuPrivs", workspaceMenuPrivs); + } + + @RequestMapping(path ="getWorkspaceHomepageSettings", method = RequestMethod.GET) + public Message getWorkspaceHomepageSettings(HttpServletRequest request, @RequestParam(WORKSPACE_ID_STR) int workspaceId){ + String username = SecurityFilter.getLoginUsername(request); + DSSWorkspaceHomepageSettingVO dssWorkspaceHomepageSettingVO = dssWorkspaceService.getWorkspaceHomepageSettings(workspaceId); + return Message.ok().data("homepageSettings", dssWorkspaceHomepageSettingVO); + } + + @RequestMapping(path ="updateRoleMenuPriv", method = RequestMethod.POST) + public Message updateRoleMenuPriv(HttpServletRequest request,@RequestBody UpdateRoleMenuPrivRequest updateRoleMenuPrivRequest){ + String updater = SecurityFilter.getLoginUsername(request); + int menuId = updateRoleMenuPrivRequest.getMenuId(); + int workspaceId = updateRoleMenuPrivRequest.getWorkspaceId(); + Map menuPrivs = updateRoleMenuPrivRequest.getMenuPrivs(); + List> pairs = new ArrayList<>(); + for(String key : menuPrivs.keySet()){ + Integer roleId = dssWorkspacePrivService.getRoleId(workspaceId, key); + if (roleId == null) { + roleId = workspaceDBHelper.getRoleIdByName(key); + } + pairs.add(new Pair(roleId, menuPrivs.get(key))); + } + dssWorkspacePrivService.updateRoleMenuPriv(workspaceId, menuId, updater, pairs); + return Message.ok("更新角色对于菜单的权限成功"); + } + + @RequestMapping(path ="updateRoleComponentPriv", method = RequestMethod.POST) + public Message updateRoleComponentPriv(HttpServletRequest request,@RequestBody UpdateRoleComponentPrivRequest updateRoleComponentPrivRequest){ + //todo 更新工作空间中角色对于component的权限 + String username = SecurityFilter.getLoginUsername(request); + int menuId = updateRoleComponentPrivRequest.getComponentId(); + int workspaceId = updateRoleComponentPrivRequest.getWorkspaceId(); + Map componentPrivs = updateRoleComponentPrivRequest.getComponentPrivs(); + List> pairs = new ArrayList<>(); + for (String key : componentPrivs.keySet()){ + Integer roleId = dssWorkspacePrivService.getRoleId(workspaceId, key); + if (roleId == null) { + roleId = workspaceDBHelper.getRoleIdByName(key); + } + pairs.add(new Pair(roleId, componentPrivs.get(key))); + } + dssWorkspacePrivService.updateRoleComponentPriv(workspaceId, menuId, username, pairs); + return Message.ok().data("updateRoleComponentPriv","更新组件权限成功"); + } + + @RequestMapping(path ="updateRoleHomepage", method = RequestMethod.POST) + public Message updateRoleHomepage(HttpServletRequest request){ + return null; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRestful.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRestful.java new file mode 100644 index 000000000..659eb716d --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRestful.java @@ -0,0 +1,126 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.restful; + +import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace; +import com.webank.wedatasphere.dss.framework.workspace.bean.request.CreateWorkspaceRequest; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceHomePageVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceOverviewVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DepartmentVO; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardWarnException; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.List; + +import static com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant.WORKSPACE_ID_STR; + +@RequestMapping(path = "/dss/framework/workspace", produces = {"application/json"}) +@RestController +public class DSSWorkspaceRestful { + @Autowired + private DSSWorkspaceService dssWorkspaceService; + @Autowired + private DssAdminUserService dssUserService; + @Autowired + private WorkspaceDBHelper workspaceDBHelper; + + @RequestMapping(path ="createWorkspace", method = RequestMethod.POST) + public Message createWorkspace(HttpServletRequest request, @RequestBody CreateWorkspaceRequest createWorkspaceRequest)throws ErrorException { + String userName = SecurityFilter.getLoginUsername(request); + if (!dssWorkspaceService.checkAdmin(userName)){ + return Message.error("您好,您不是管理员,没有权限建立工作空间"); + } + String workSpaceName = createWorkspaceRequest.getWorkspaceName(); + String department = createWorkspaceRequest.getDepartment(); + String description = createWorkspaceRequest.getDescription(); + String stringTags = createWorkspaceRequest.getTags(); + String productName = createWorkspaceRequest.getProductName(); + int workspaceId = dssWorkspaceService.createWorkspace(workSpaceName, stringTags, userName, description, department, productName,""); + return Message.ok().data("workspaceId", workspaceId).data("workspaceName",workSpaceName); + } + + @RequestMapping(path ="listDepartments", method = RequestMethod.GET) + public Message listDepartments(HttpServletRequest request, @RequestParam(WORKSPACE_ID_STR) String workspaceId){ + //todo 要从um中获取 + List departments = dssWorkspaceService.getDepartments(); + return Message.ok().data("departments", departments); + } + + @RequestMapping(path ="getWorkspaces", method = RequestMethod.GET) + public Message getWorkspaces(HttpServletRequest request){ + String username = SecurityFilter.getLoginUsername(request); + List workspaces = dssWorkspaceService.getWorkspaces(username); + List dssWorkspaceVOS = new ArrayList<>(); + for (DSSWorkspace workspace:workspaces ){ + String name = workspace.getName(); + int id = workspace.getId(); + String labels = workspace.getLabel(); + DSSWorkspaceVO dssWorkspaceVO = new DSSWorkspaceVO(); + dssWorkspaceVO.setId(id); + dssWorkspaceVO.setName(name); + dssWorkspaceVO.setTags(labels); + dssWorkspaceVO.setDepartment(workspace.getDepartment()); + dssWorkspaceVO.setDescription(workspace.getDescription()); + dssWorkspaceVO.setProduct(workspace.getProduct()); + dssWorkspaceVOS.add(dssWorkspaceVO); + } + //todo 获取用户所有所有能够访问的工作空间 + return Message.ok().data("workspaces", dssWorkspaceVOS); + } + + @RequestMapping(path ="getWorkspaceHomePage", method = RequestMethod.GET) + public Message getWorkspaceHomePage(HttpServletRequest request, @RequestParam(required = false, name = "micro_module") String moduleName) throws Exception{ + //如果用户的工作空间大于两个,那么就直接返回/workspace页面 + String username = SecurityFilter.getLoginUsername(request); + Workspace workspace = new Workspace(); + try { + SSOHelper.addWorkspaceInfo(request, workspace); + } catch (AppStandardWarnException ignored) {} // ignore it. + dssUserService.insertOrUpdateUser(username, workspace); + DSSWorkspaceHomePageVO dssWorkspaceHomePageVO = dssWorkspaceService.getWorkspaceHomePage(username,moduleName); + return Message.ok().data("workspaceHomePage", dssWorkspaceHomePageVO); + } + + @RequestMapping(path ="getOverview", method = RequestMethod.GET) + public Message getOverview(HttpServletRequest request, @RequestParam(WORKSPACE_ID_STR) int workspaceId){ + String username = SecurityFilter.getLoginUsername(request); + String language = request.getHeader("Content-language"); + boolean isEnglish = "en".equals(language); + DSSWorkspaceOverviewVO dssWorkspaceOverviewVO = dssWorkspaceService.getOverview(username, workspaceId, isEnglish); + return Message.ok().data("overview", dssWorkspaceOverviewVO); + } + + @RequestMapping(path ="refreshCache", method = RequestMethod.GET) + public Message refreshCache(HttpServletRequest request){ + workspaceDBHelper.retrieveFromDB(); + return Message.ok("refresh ok"); + } + +} + diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRoleRestful.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRoleRestful.java new file mode 100644 index 000000000..5f383e0cc --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRoleRestful.java @@ -0,0 +1,134 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.restful; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.request.AddWorkspaceRoleRequest; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceRoleVO; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import java.util.List; + +import static com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant.WORKSPACE_ID_STR; + + +@RequestMapping(path = "/dss/framework/workspace", produces = {"application/json"}) +@RestController +public class DSSWorkspaceRoleRestful { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSWorkspaceRoleRestful.class); + @Autowired + DSSWorkspaceService dssWorkspaceService; + @Autowired + private DSSWorkspaceRoleService dssWorkspaceRoleService; + + @RequestMapping(path = "getWorkspaceRoles", method = RequestMethod.GET) + public Message getWorkspaceRoles(HttpServletRequest request, @RequestParam(WORKSPACE_ID_STR) int workspaceId) { + //todo 获取工作空间中所有的角色 + List workspaceRoles = dssWorkspaceService.getWorkspaceRoles(workspaceId); + return Message.ok().data("workspaceRoles", workspaceRoles); + } + + @RequestMapping(path = "addWorkspaceRole", method = RequestMethod.POST) + public Message addWorkspaceRole(HttpServletRequest request, @RequestBody AddWorkspaceRoleRequest addWorkspaceRoleRequest) { + String username = SecurityFilter.getLoginUsername(request); + int workspaceId = addWorkspaceRoleRequest.getWorkspaceId(); + String roleName = addWorkspaceRoleRequest.getRoleName(); + List menuIds = addWorkspaceRoleRequest.getMenuIds(); + List componentIds = addWorkspaceRoleRequest.getComponentIds(); + if (!dssWorkspaceService.checkAdmin(username) || !dssWorkspaceService.checkAdminByWorkspace(username, workspaceId)) { + return Message.error("无权限进行该操作"); + } + dssWorkspaceRoleService.addWorkspaceRole(roleName, workspaceId, menuIds, componentIds, username); + return Message.ok("创建角色成功"); + } + + @RequestMapping(path = "deleteWorkspaceRole", method = RequestMethod.POST) + public Message deleteWorkspaceRole(HttpServletRequest request) { + + return null; + } + +// @RequestMapping(path = "getWorkspaceBaseInfo", method = RequestMethod.GET) +// @Deprecated +// public Message getWorkspaceInfo(HttpServletRequest request, +// HttpServletResponse response, +// @RequestParam(WORKSPACE_ID_STR) Integer workspaceId) { +// String username = SecurityFilter.getLoginUsername(request); +// //如果workspaceId为null的话,那么就找到这个用户工作空间 +// if (workspaceId == null || workspaceId <= 0) { +// workspaceId = dssWorkspaceRoleService.getWorkspaceIdByUser(username); +// } +// DSSWorkspace workspace; +// try { +// workspace = dssWorkspaceService.getWorkspacesById(workspaceId.longValue(), username); +// } catch (DSSErrorException e) { +// return Message.error(ExceptionUtils.getRootCauseMessage(e)); +// } +// //将workspaceId作为cookie写入 +// SSOHelper.setAndGetWorkspace(request, response, workspaceId, workspace.getName()); +// List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, workspaceId); +// if (roles == null || roles.isEmpty()) { +// LOGGER.error("username {}, in workspace {} roles are null or empty", username, workspaceId); +// return Message.error("can not get roles information"); +// } +// //判断如果是没有权限的,那么就直接干掉 +// if (roles.contains("apiUser")) { +// int priv = dssWorkspaceRoleService.getApiPriv(username, workspaceId, "apiUser", "apiService"); +// if (priv <= 0) { +// roles.remove("apiUser"); +// } +// } +// Message retMessage = Message.ok(); +// //工作空间中,加上用户在顶部的菜单 +// if (roles.contains("analyser")) { +// retMessage.data("topName", "Scriptis"); +// retMessage.data("topUrl", "/home"); +// } else if (roles.contains("developer")) { +// retMessage.data("topName", "Scriptis"); +// retMessage.data("topUrl", "/home"); +// } else if (roles.contains("apiUser") && roles.size() == 1) { +// retMessage.data("topName", "Scriptis"); +// retMessage.data("topUrl", "/home"); +// } else { +// retMessage.data("topName", "Scriptis"); +// retMessage.data("topUrl", "/home"); +// } +// //如果其他的角色也是有这个api权限的,那么就加上这个apiUser +// boolean flag = false; +// for (String role : roles) { +// int priv = dssWorkspaceRoleService.getApiPriv(username, workspaceId, role, "apiService"); +// if (priv >= 1) { +// flag = true; +// break; +// } +// } +// if (flag && !roles.contains("apiUser")) { +// roles.add("apiUser"); +// } +// return retMessage.data("roles", roles).data("workspaceId", workspaceId); +// } + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestful.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestful.java new file mode 100644 index 000000000..caab1675c --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestful.java @@ -0,0 +1,177 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.restful; + +import com.webank.wedatasphere.dss.framework.workspace.bean.request.DeleteWorkspaceUserRequest; +import com.webank.wedatasphere.dss.framework.workspace.bean.request.UpdateWorkspaceUserRequest; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceRoleVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceUserVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceUsersVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.StaffInfoVO; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService; +import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant.WORKSPACE_ID_STR; + + +@RequestMapping(path = "/dss/framework/workspace", produces = {"application/json"}) +@RestController +public class DSSWorkspaceUserRestful { + private static final Logger LOGGER = LoggerFactory.getLogger(DSSWorkspaceUserRestful.class); + + @Autowired + private DSSWorkspaceService dssWorkspaceService; + @Autowired + private WorkspaceDBHelper workspaceDBHelper; + @Autowired + private DSSWorkspaceUserService dssWorkspaceUserService; + + @RequestMapping(path = "getWorkspaceUsers", method = RequestMethod.GET) + public Message getWorkspaceUsers(HttpServletRequest request, @RequestParam(WORKSPACE_ID_STR) String workspaceId, + @RequestParam(required = false, name = "pageNow") Integer pageNow, @RequestParam(required = false, name = "pageSize") Integer pageSize, + @RequestParam(required = false, name = "department") String department, @RequestParam(required = false, name = "userName") String username, + @RequestParam(required = false, name = "roleName") String roleName) { + //todo 获取工作空间中所有的用户以及他们的角色信息 + if (pageNow == null) { + pageNow = 1; + } + if (pageSize == null) { + //默认改成20 + pageSize = 20; + } + if (StringUtils.isNotEmpty(roleName)) { + //如果roleName不是空的话,就按照roleName来吧 + List totals = new ArrayList<>(); + List workspaceUsers = + dssWorkspaceService.getWorkspaceUsersByRole(Integer.parseInt(workspaceId), roleName, totals, pageNow, pageSize); + List dssRoles = workspaceDBHelper.getRoleVOs(Integer.parseInt(workspaceId)); + return Message.ok().data("roles", dssRoles).data("workspaceUsers", workspaceUsers).data("total", totals.get(0)); + } else { + List totals = new ArrayList<>(); + List workspaceUsers = + dssWorkspaceService.getWorkspaceUsers(workspaceId, department, username, roleName, pageNow, pageSize, totals); + List dssRoles = workspaceDBHelper.getRoleVOs(Integer.parseInt(workspaceId)); + return Message.ok().data("roles", dssRoles).data("workspaceUsers", workspaceUsers).data("total", totals.get(0)); + } + } + + @RequestMapping(path = "getAllWorkspaceUsers", method = RequestMethod.GET) + public Message getAllWorkspaceUsers(HttpServletRequest request) { + DSSWorkspaceUsersVo dssWorkspaceUsersVo = new DSSWorkspaceUsersVo(); + // workspaceId改为从cookie取 + int workspaceId = (int) SSOHelper.getWorkspace(request).getWorkspaceId(); + dssWorkspaceUsersVo.setAccessUsers(dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId)); +// dssWorkspaceUsersVo.setEditUsers(dssWorkspaceUserService.getWorkspaceEditUsers(workspaceId)); +// dssWorkspaceUsersVo.setReleaseUsers(dssWorkspaceUserService.getWorkspaceReleaseUsers(workspaceId)); + dssWorkspaceUsersVo.setEditUsers(dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId)); + dssWorkspaceUsersVo.setReleaseUsers(dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId)); + return Message.ok().data("users", dssWorkspaceUsersVo); + } + + + @RequestMapping(path = "existUserInWorkspace", method = RequestMethod.GET) + public Message existUserInWorkspace(HttpServletRequest request, @RequestParam(WORKSPACE_ID_STR) int workspaceId, @RequestParam("queryUserName") String queryUserName) { + String username = SecurityFilter.getLoginUsername(request); + List users = dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId); + boolean existFlag = users.stream().anyMatch(user -> user.equalsIgnoreCase(queryUserName)); + LOGGER.info("Check exist user result:" + existFlag + ", query user is " + queryUserName + ",workSpace id is " + workspaceId); + return Message.ok().data("existFlag", existFlag); + } + + + @RequestMapping(path = "addWorkspaceUser", method = RequestMethod.POST) + public Message addWorkspaceUser(HttpServletRequest request, @RequestBody UpdateWorkspaceUserRequest updateWorkspaceUserRequest) { + //todo 工作空间添加用户 + String creator = SecurityFilter.getLoginUsername(request); + List roles = updateWorkspaceUserRequest.getRoles(); + Workspace workspace = SSOHelper.getWorkspace(request); + int workspaceId = updateWorkspaceUserRequest.getWorkspaceId(); + if (workspace.getWorkspaceId() != workspaceId) { + return Message.error("cookie 中的 workspaceId 与请求添加用户的 workspace 不同!"); + } + String userName = updateWorkspaceUserRequest.getUserName(); + String userId = updateWorkspaceUserRequest.getUserId(); + Long count = dssWorkspaceUserService.getCountByUsername(userName, workspaceId); + if (count != null && count > 0) { + return Message.error("用户已经存在该工作空间,不需要重复添加!"); + } + if (!dssWorkspaceService.isAdminUser((long) workspaceId, creator)) { + return Message.error("无权限进行该操作"); + } + dssWorkspaceService.addWorkspaceUser(roles, workspace, userName, creator, userId); + return Message.ok(); + } + + @RequestMapping(path = "updateWorkspaceUser", method = RequestMethod.POST) + public Message updateWorkspaceUser(HttpServletRequest request, @RequestBody UpdateWorkspaceUserRequest updateWorkspaceUserRequest) { + String creator = SecurityFilter.getLoginUsername(request); + List roles = updateWorkspaceUserRequest.getRoles(); + int workspaceId = updateWorkspaceUserRequest.getWorkspaceId(); + if (!dssWorkspaceService.isAdminUser(Long.valueOf(workspaceId), creator)) { + return Message.error("无权限进行该操作"); + } + String userName = updateWorkspaceUserRequest.getUserName(); + dssWorkspaceUserService.updateWorkspaceUser(roles, workspaceId, userName, creator); + return Message.ok(); + } + + @RequestMapping(path = "deleteWorkspaceUser", method = RequestMethod.POST) + public Message deleteWorkspaceUser(HttpServletRequest request, @RequestBody DeleteWorkspaceUserRequest deleteWorkspaceUserRequest) { + //todo 删除工作空间中的用户 + String userName = deleteWorkspaceUserRequest.getUserName(); + int workspaceId = deleteWorkspaceUserRequest.getWorkspaceId(); + String creator = SecurityFilter.getLoginUsername(request); + if (!dssWorkspaceService.checkAdmin(creator) || !dssWorkspaceService.checkAdminByWorkspace(creator, workspaceId)) { + return Message.error("无权限进行该操作"); + } + dssWorkspaceUserService.deleteWorkspaceUser(userName, workspaceId); + return Message.ok(); + } + + @RequestMapping(path = "listAllUsers", method = RequestMethod.GET) + public Message listAllUsers(HttpServletRequest request) { + List dssUsers = dssWorkspaceUserService.listAllDSSUsers(); + return Message.ok().data("users", dssUsers); + } + + @RequestMapping(path = "getWorkspaceIdByUserName", method = RequestMethod.GET) + public Message getWorkspaceIdByUserName(HttpServletRequest request, @RequestParam(required = false, name = "userName") String userName) { + String loginUserName = SecurityFilter.getLoginUsername(request); + String queryUserName = userName; + if (StringUtils.isEmpty(userName)) { + queryUserName = loginUserName; + } + List userWorkspaceIds = dssWorkspaceUserService.getUserWorkspaceIds(queryUserName); + String userWorkspaceIdStr = userWorkspaceIds.stream().map(x -> x.toString()).collect(Collectors.joining(",")); + return Message.ok().data("userWorkspaceIds", userWorkspaceIdStr); + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/WorkspaceRestfulApi.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/WorkspaceRestfulApi.java new file mode 100644 index 000000000..c24b5e26f --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/WorkspaceRestfulApi.java @@ -0,0 +1,227 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.restful; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceFavoriteVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceMenuVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DepartmentVO; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@RestController +@RequestMapping(path = "/dss/framework/workspace", produces = {"application/json"}) +public class WorkspaceRestfulApi { + private static final Logger LOGGER = LoggerFactory.getLogger(WorkspaceRestfulApi.class); + + @Autowired + private DSSWorkspaceService dssWorkspaceService; + + @Autowired + private DSSWorkspaceRoleService dssWorkspaceRoleService; + + @RequestMapping(path = "workspaces", method = RequestMethod.GET) + public Message getAllWorkspaces(HttpServletRequest req) { + String username = SecurityFilter.getLoginUsername(req); + List workspaces = dssWorkspaceService.getWorkspaces(username); + return Message.ok().data("workspaces", workspaces); + } + + /** + * 获取所有工程或者单个工程 + * + * @param request + * @return + */ + @RequestMapping(path = "getWorkSpaceStr", method = RequestMethod.GET) + public Message getWorkSpaceStr(HttpServletRequest request, + HttpServletResponse response, + @RequestParam(name = "workspaceName") String workspaceName) { + String username = SecurityFilter.getLoginUsername(request); + DSSWorkspace workspaceEntity; + try { + workspaceEntity = dssWorkspaceService.getWorkspacesByName(workspaceName, username); + } catch (DSSErrorException e) { + LOGGER.error("User {} get workspace {} failed.", username, workspaceName, e); + return Message.error(e); + } + Workspace workspace = SSOHelper.setAndGetWorkspace(request, response, workspaceEntity.getId(), workspaceName); + return Message.ok("succeed.").data("workspaceStr", DSSCommonUtils.COMMON_GSON.toJson(workspace)); + } + + @RequestMapping(path = "/workspaces/{id}", method = RequestMethod.GET) + public Message getWorkspacesById(HttpServletRequest req, + HttpServletResponse resp, + @PathVariable("id") Long workspaceId) { + String username = SecurityFilter.getLoginUsername(req); + DSSWorkspace workspace = null; + try { + workspace = dssWorkspaceService.getWorkspacesById(workspaceId, username); + } catch (DSSErrorException e) { + LOGGER.error("User {} get workspace {} failed.", username, workspaceId, e); + return Message.error(e); + } + SSOHelper.setAndGetWorkspace(req, resp, workspace.getId(), workspace.getName()); + List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, workspaceId.intValue()); + if (roles == null || roles.isEmpty()) { + LOGGER.error("username {}, in workspace {} roles are null or empty", username, workspaceId); + return Message.error("can not get roles information"); + } + //判断如果是没有权限的,那么就直接干掉 + if (roles.contains("apiUser")) { + int priv = dssWorkspaceRoleService.getApiPriv(username, workspaceId.intValue(), "apiUser", "apiService"); + if (priv <= 0) { + roles.remove("apiUser"); + } + } + Message retMessage = Message.ok(); + //如果其他的角色也是有这个api权限的,那么就加上这个apiUser + boolean flag = false; + for (String role : roles) { + int priv = dssWorkspaceRoleService.getApiPriv(username, workspaceId.intValue(), role, "apiService"); + if (priv >= 1) { + flag = true; + break; + } + } + if (flag && !roles.contains("apiUser")) { + roles.add("apiUser"); + } + return retMessage.data("roles", roles).data("workspace", workspace); + } + + @RequestMapping(path = "/workspaces/departments", method = RequestMethod.GET) + public Message getAllWorkspaceDepartments(HttpServletRequest req) { + List departments = dssWorkspaceService.getDepartments(); + return Message.ok().data("departments", departments); + } + + @RequestMapping(path = "/workspaces/exists", method = RequestMethod.GET) + public Message getUsernameExistence(HttpServletRequest req, @RequestParam(required = false, name = "name") String name) { + boolean exists = dssWorkspaceService.existWorkspaceName(name); + return Message.ok().data("workspaceNameExists", exists); + } + + @RequestMapping(path = "/workspaces", method = RequestMethod.POST) + public Message addWorkspace(HttpServletRequest req, @RequestBody Map json) throws ErrorException { + String userName = SecurityFilter.getLoginUsername(req); + if (!dssWorkspaceService.checkAdmin(userName)) { + return Message.error("您好,您不是管理员,没有权限建立工作空间"); + + } + String name = json.get("name"); + if (dssWorkspaceService.existWorkspaceName(name)) { + return Message.error("工作空间名重复"); + } + String department = json.get("department"); + String label = json.get("label"); + String description = json.get("description"); + String workspaceType = json.get("workspace_type"); + + String productName = "DSS"; + int workspaceId = dssWorkspaceService.createWorkspace(name, label, userName, description, department, productName, workspaceType); + return Message.ok().data("workspaceId", workspaceId); + } + + @RequestMapping(path = "workspaces/{workspaceId}/appconns", method = RequestMethod.GET) + public Message getWorkspaceAppConns(HttpServletRequest req, @PathVariable("workspaceId") Long workspaceId) { + String header = req.getHeader("Content-language").trim(); + boolean isChinese = "zh-CN".equals(header); + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + List appconns; + try { + appconns = dssWorkspaceService.getWorkspaceAppConns(workspace, workspaceId, username, isChinese); + } catch (DSSErrorException e) { + LOGGER.warn("{} get appconns from workspace {} failed.", username, workspaceId, e); + return Message.error(e); + } + return Message.ok().data("menus", appconns); + } + + + @RequestMapping(path = "/workspaces/{workspaceId}/favorites", method = RequestMethod.GET) + public Message getWorkspaceFavorites(HttpServletRequest req, @PathVariable("workspaceId") Long workspaceId, @RequestParam(value = "type", required = false) String type) { + String header = req.getHeader("Content-language").trim(); + boolean isChinese = "zh-CN".equals(header); + String username = SecurityFilter.getLoginUsername(req); + List favorites = dssWorkspaceService.getWorkspaceFavorites(workspaceId, username, isChinese, type == null ? "" : type); + Set favoriteVos = new HashSet<>(favorites); + return Message.ok().data("favorites", favoriteVos); + } + + /** + * 应用加入收藏,返回收藏后id + * + * @param req + * @param json + * @return + */ + @RequestMapping(path = "/workspaces/{workspaceId}/favorites", method = RequestMethod.POST) + public Message addFavorite(HttpServletRequest req, @PathVariable("workspaceId") Long workspaceId, @RequestBody Map json) { + String username = SecurityFilter.getLoginUsername(req); +// Long menuApplicationId = json.get("menuApplicationId").getLongValue(); + Long menuApplicationId = Long.valueOf(json.get("menuApplicationId")); + String type = json.get("type") == null ? "" : json.get("type"); + Long favoriteId = dssWorkspaceService.addFavorite(username, workspaceId, menuApplicationId, type); + return Message.ok().data("favoriteId", favoriteId); + } + + + @RequestMapping(path = "/workspaces/{workspaceId}/favorites/{applicationId}", method = RequestMethod.POST) + public Message deleteFavorite(HttpServletRequest req, @PathVariable("workspaceId") Long workspaceId, @PathVariable("applicationId") Long applicationId, @RequestBody Map json) { + String username = SecurityFilter.getLoginUsername(req); + String type = json.get("type") == null ? "" : json.get("type"); + Long favoriteId = dssWorkspaceService.deleteFavorite(username, applicationId, workspaceId, type); + return Message.ok().data("favoriteId", favoriteId); + } + + + @RequestMapping(path = "/workspaces/{workspaceId}/favorites/{applicationId}", method = RequestMethod.GET) + public Message deleteFavorite1(HttpServletRequest req, @PathVariable("workspaceId") Long workspaceId, @PathVariable("applicationId") Long applicationId, @RequestParam(value = "type", required = false) String type) { + String username = SecurityFilter.getLoginUsername(req); + Long favoriteId = dssWorkspaceService.deleteFavorite(username, applicationId, workspaceId, type); + return Message.ok().data("favoriteId", favoriteId); + } + + @RequestMapping(path = "/workspaces/{workspaceId}/favorites/{favouritesId}", method = RequestMethod.DELETE) + public Message deleteFavorite2(HttpServletRequest req, @PathVariable("workspaceId") Long workspaceId, @PathVariable("favouritesId") Long favouritesId, @RequestParam(value = "type", required = false) String type) { + String username = SecurityFilter.getLoginUsername(req); + Long favoriteId = dssWorkspaceService.deleteFavorite(username, favouritesId, workspaceId, type); + return Message.ok().data("favoriteId", favoriteId); + } + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSDictionaryService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSDictionaryService.java new file mode 100644 index 000000000..ab71a8185 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSDictionaryService.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSDictionary; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSDictionaryRequestVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DepartmentVO; + +import java.util.List; +import java.util.Map; + + +public interface DSSDictionaryService { + + public List getListByParam(DSSDictionaryRequestVO dictionaryRequestVO) ; + + /** + * 获取编排模式 + * @param dictionaryRequestVO + * @return + */ + public Map getDicSecondList(DSSDictionaryRequestVO dictionaryRequestVO) ; + + /** + * 获取空间默认部门 + * @return + */ + public List getDefaultDepartmentVOList(); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSSideInfoService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSSideInfoService.java new file mode 100644 index 000000000..1d977e8e2 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSSideInfoService.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.SidebarVO; + +import java.util.List; + + +public interface DSSSideInfoService { + + List getSidebarVOList(String username, Long workspaceId,boolean isEnglish); +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceMenuService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceMenuService.java new file mode 100644 index 000000000..dfdf49c50 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceMenuService.java @@ -0,0 +1,29 @@ +///* +// * Copyright 2019 WeBank +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// * +// */ +// +//package com.webank.wedatasphere.dss.framework.workspace.service; +// +// +// +//import com.webank.wedatasphere.dss.framework.workspace.bean.DSSMenu; +// +// +//import java.util.List; +// +//public interface DSSWorkspaceMenuService { +// @Deprecated +// List getRealComponents(List subMenus, String workspaceId, String username); +//} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspacePrivService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspacePrivService.java new file mode 100644 index 000000000..c964d0e50 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspacePrivService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service; + +import org.apache.commons.math3.util.Pair; + +import java.util.List; + + +public interface DSSWorkspacePrivService { + void updateRoleMenuPriv(int workspaceId, int menuId, String updater, List> pairs); + + void updateRoleComponentPriv(int workspaceId, int menuId, String username, List> pairs); + + Integer getRoleId(int workspaceId, String key); +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceRoleService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceRoleService.java new file mode 100644 index 000000000..c8de173f9 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceRoleService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service; + +import java.util.List; + + +public interface DSSWorkspaceRoleService { + void addWorkspaceRole(String roleName, int workspaceId, List menuIds, List componentIds, String username); + + List getRoleInWorkspace(String username, int workspaceId); + + Integer getWorkspaceIdByUser(String username); + + int getApiPriv(String username, Integer workspaceId, String roleName, String appName); +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceService.java new file mode 100644 index 000000000..ba2c1587a --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceService.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceMenuVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceDepartmentVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceFavoriteVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceHomePageVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceHomepageSettingVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceOverviewVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspacePrivVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceRoleVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceUserVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DepartmentVO; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.List; + + +public interface DSSWorkspaceService { + + + int createWorkspace(String workspaceName, String tags, String userName, String description, String department, String productName,String workspaceType) throws ErrorException; + + void addWorkspaceUser(List roleIds, Workspace workspace, String userName, String creater, String userId); + + List getWorkspaces(String userName); + + DSSWorkspaceHomePageVO getWorkspaceHomePage(String userName,String moduleName) throws DSSErrorException; + + List getWorkspaceUsers(String workspaceId, String department, String username, + String roleName, int pageNow, int pageSize, List total); + + + List getWorkspaceRoles(int workspaceId); + + DSSWorkspacePrivVO getWorkspaceMenuPrivs(String workspaceId); + + DSSWorkspaceOverviewVO getOverview(String username, int workspaceId, boolean isEnglish); + + DSSWorkspaceHomepageSettingVO getWorkspaceHomepageSettings(int workspaceId); + + String getWorkspaceName(String workspaceId); + + boolean checkAdmin(String userName); + + List getDepartments(); + + List getWorkspaceUsersByRole(int workspaceId, String roleName, List totals, + int pageNow, int pageSize); + + Long addWorkspace(String userName, String name, String department, String label, String description); + + boolean existWorkspaceName(String name); + + List getWorkSpaceDepartments(); + + List getWorkspaceAppConns(Workspace workspace, Long workspaceId, + String username, boolean isChinese) throws DSSErrorException; + + DSSWorkspace getWorkspacesById(Long id, String username) throws DSSErrorException; + + DSSWorkspace getWorkspacesByName(String workspaceName, String username) throws DSSErrorException; + + List getWorkspaceFavorites(Long workspaceId, String username, boolean isChinese,String type); + + Long addFavorite(String username, Long workspaceId, Long menuApplicationId,String type); + + Long deleteFavorite(String username, Long applicationId, Long workspaceId,String type); + + + boolean checkAdminByWorkspace(String username, int workspaceId); + + //是否为超级管理员 + public boolean isAdminUser(Long workspaceId, String username); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceUserService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceUserService.java new file mode 100644 index 000000000..30bcb8941 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceUserService.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service; + + +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.StaffInfoVO; + +import java.util.List; + + +public interface DSSWorkspaceUserService { + + void updateWorkspaceUser(List roles, int workspaceId, String userName, String creator); + + void deleteWorkspaceUser(String userName, int workspaceId); + + List listAllDSSUsers(); + + List getAllWorkspaceUsers(int workspaceId); + + List getUserWorkspaceIds(String userName); + + List getWorkspaceEditUsers(int workspaceId); + + List getWorkspaceReleaseUsers(int workspaceId); + + Long getCountByUsername(String username,int workspaceId); +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/StaffInfoGetter.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/StaffInfoGetter.java new file mode 100644 index 000000000..1d5468d5b --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/StaffInfoGetter.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service; + +import com.webank.wedatasphere.dss.framework.workspace.bean.StaffInfo; + +import java.util.List; + + +public interface StaffInfoGetter { + + List getAllUsers(); + + String getFullOrgNameByUsername(String username); + + List getAllDepartments(); + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSDictionaryServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSDictionaryServiceImpl.java new file mode 100644 index 000000000..88e3980bd --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSDictionaryServiceImpl.java @@ -0,0 +1,120 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSDictionary; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSDictionaryRequestVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DepartmentVO; +import com.webank.wedatasphere.dss.framework.workspace.dao.DSSDictionaryMapper; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSDictionaryService; +import com.webank.wedatasphere.dss.framework.workspace.util.DSSDictionaryConstant; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.stream.Collectors; + + +@Service +public class DSSDictionaryServiceImpl implements DSSDictionaryService { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSDictionaryServiceImpl.class); + + @Autowired + private DSSDictionaryMapper dssDictionaryMapper; + + @Override + public List getListByParam(DSSDictionaryRequestVO dictionaryRequestVO) { + //查询所在工作空间以及默认空间的开发流程或编码模式配置 + QueryWrapper dictionaryQueryWrapper = new QueryWrapper<>(); + dictionaryQueryWrapper.in("workspace_id", Arrays.asList(dictionaryRequestVO.getWorkspaceId(),0)); + if(StringUtils.isNotBlank(dictionaryRequestVO.getDicKey())){ + dictionaryQueryWrapper.eq("dic_key",dictionaryRequestVO.getDicKey()); + } + if(StringUtils.isNotBlank(dictionaryRequestVO.getParentKey())){ + dictionaryQueryWrapper.eq("parent_key",dictionaryRequestVO.getParentKey()); + } + dictionaryQueryWrapper.orderByAsc("order_num"); + List dictionaries = dssDictionaryMapper.selectList(dictionaryQueryWrapper); + return dictionaries; + } + + @Override + public Map getDicSecondList(DSSDictionaryRequestVO dictionaryRequestVO) { + //一级字典 + List dictionaries = getListByParam(dictionaryRequestVO); + + //二级字典 + List keyList = dictionaries.stream().map(DSSDictionary::getDicKey).collect(Collectors.toList()); + QueryWrapper dictionaryQueryWrapper = new QueryWrapper<>(); + dictionaryQueryWrapper.in("workspace_id", Arrays.asList(dictionaryRequestVO.getWorkspaceId(),0)); + dictionaryQueryWrapper.in("parent_key",keyList); + dictionaryQueryWrapper.orderByAsc("order_num"); + List dictionarieSelectList = dssDictionaryMapper.selectList(dictionaryQueryWrapper); + + //封装返回 + Map> mapList = new HashMap<>(); + for(DSSDictionary dssDictionary : dictionarieSelectList){ + String parentKey = dssDictionary.getParentKey(); + if(mapList.get(parentKey)==null){ + mapList.put(parentKey,new ArrayList<>()); + } + mapList.get(parentKey).add(dssDictionary); + } + Map retMap = new HashMap<>(); + retMap.put("list",dictionaries); + retMap.put("mapList",mapList); + return retMap; + } + + + /** + * 获取空间默认部门 + * @return + */ + @Override + public List getDefaultDepartmentVOList(){ + QueryWrapper dictionaryQueryWrapper = new QueryWrapper<>(); + dictionaryQueryWrapper.eq("workspace_id", 0); + dictionaryQueryWrapper.eq("dic_key", DSSDictionaryConstant.W_WORKSPACE_DEPARTMENT); + List dictionarieList = dssDictionaryMapper.selectList(dictionaryQueryWrapper); + List retList = new ArrayList<>(); + if(CollectionUtils.isEmpty(dictionarieList)){ + return retList; + } + String dicValue = dictionarieList.get(0).getDicValue(); + if(StringUtils.isBlank(dicValue)){ + return retList; + } + String[] tempStrArr = dicValue.split(";"); + DepartmentVO departmentVO = null; + for(int i=0;i getSidebarVOList(String username, Long workspaceId,boolean isEnglish) { + //查询所在工作空间以及默认空间的侧边栏 + List retList = new ArrayList<>(); + QueryWrapper sidebarQueryWrapper = new QueryWrapper<>(); + sidebarQueryWrapper.in("workspace_id", Arrays.asList(workspaceId,0)); + sidebarQueryWrapper.orderByAsc("order_num"); + List sidebarList = sidebarMapper.selectList(sidebarQueryWrapper); + if(CollectionUtils.isEmpty(sidebarList)){ + return retList; + } + //查询所在工作空间以及默认空间的侧边栏对应的侧边栏-内容 + List sideBarIds = sidebarList.stream().map(Sidebar::getId).collect(Collectors.toList()); + QueryWrapper sidebarQueryContentWrapper = new QueryWrapper<>(); + sidebarQueryContentWrapper.in("workspace_id", Arrays.asList(workspaceId.intValue(),0)); + sidebarQueryContentWrapper.in("sidebar_id", sideBarIds); + sidebarQueryContentWrapper.orderByAsc("order_num"); + List sidebarContentList = sidebarContentMapper.selectList(sidebarQueryContentWrapper); + + //封装返回数据 + return getSidebarVOS(username, workspaceId,retList, sidebarList, sidebarContentList,isEnglish); + } + + //封装返回数据 + private List getSidebarVOS(String username, Long workspaceId,List retList, List sidebarList, List sidebarContentList,boolean isEnglish) { + Map> contentMap = new HashMap<>(); + for(SidebarContent sidebarContent : sidebarContentList){ + Integer sidebarId = sidebarContent.getSidebarId(); + if(contentMap.get(sidebarId)==null){ + contentMap.put(sidebarId,new ArrayList<>()); + } + contentMap.get(sidebarId).add(sidebarContent); + } + for (Sidebar sidebar : sidebarList){ + SidebarVO sidebarVO = new SidebarVO(); + BeanUtils.copyProperties(sidebar,sidebarVO); + international(isEnglish,sidebar, sidebarVO); + List sidebarContentVOList = new ArrayList<>(); + List sidebarContents = contentMap.get(sidebar.getId()); + if(!CollectionUtils.isEmpty(sidebarContents)){ + for (SidebarContent sidebarContent : sidebarContents){ + if(DSSWorkspaceConstant.WORKSPACE_MANAGEMENT_NAME.equals(sidebarContent.getTitle())){ + if(!workspaceService.isAdminUser(workspaceId,username)){ + continue; + } + } + SidebarContentVO sidebarContentVO = new SidebarContentVO(); + BeanUtils.copyProperties(sidebarContent,sidebarContentVO); + international(isEnglish,sidebarContent,sidebarContentVO); + sidebarContentVOList.add(sidebarContentVO); + } + } + sidebarVO.setContents(sidebarContentVOList); + retList.add(sidebarVO); + } + return retList; + } + + //国际化 + public void international(boolean isEnglish,Sidebar sidebar, SidebarVO sidebarVO){ + if(sidebar==null||!isEnglish){ + return; + } + if(StringUtils.isNotBlank(sidebar.getNameEn())){ + sidebarVO.setName(sidebar.getNameEn()); + } + if(StringUtils.isNotBlank(sidebar.getTitleEn())){ + sidebarVO.setTitle(sidebar.getTitleEn()); + } + } + + //国际化 + public void international(boolean isEnglish,SidebarContent sidebarContent, SidebarContentVO sidebarContentVO){ + if(sidebarContent==null||!isEnglish){ + return; + } + if(StringUtils.isNotBlank(sidebarContent.getNameEn())){ + sidebarContentVO.setName(sidebarContent.getNameEn()); + } + if(StringUtils.isNotBlank(sidebarContent.getTitleEn())){ + sidebarContentVO.setTitle(sidebarContent.getTitleEn()); + } + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceMenuServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceMenuServiceImpl.java new file mode 100644 index 000000000..f808953c5 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceMenuServiceImpl.java @@ -0,0 +1,64 @@ +///* +// * Copyright 2019 WeBank +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// * +// */ +// +//package com.webank.wedatasphere.dss.framework.workspace.service.impl; +// +//import com.webank.wedatasphere.dss.framework.workspace.bean.DSSMenu; +//import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspaceMenuMapper; +//import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspaceUserMapper; +//import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceMenuService; +//import org.springframework.beans.factory.annotation.Autowired; +//import org.springframework.stereotype.Service; +// +//import java.util.ArrayList; +//import java.util.HashSet; +//import java.util.List; +//import java.util.Set; +// +// +//@Service +//@Deprecated +//public class DSSWorkspaceMenuServiceImpl implements DSSWorkspaceMenuService { +// +// +// @Autowired +// private DSSWorkspaceMenuMapper dssWorkspaceMenuMapper; +// +// @Autowired +// private DSSWorkspaceUserMapper dssWorkspaceUserMapper; +// +// @Override +// public List getRealComponents(List subMenus, String workspaceId, String username) { +// Set dssMenuSet = new HashSet<>(subMenus); +// subMenus.forEach(subMenu -> { +// if (subMenu.isComponent()){ +// int applicationId = subMenu.getApplicationId(); +// List roles = dssWorkspaceUserMapper.getRoleInWorkspace(Integer.parseInt(workspaceId),username); +// int priv = 0; +// for (Integer role : roles) { +// Integer rolePriv = dssWorkspaceMenuMapper.getOneCompoentRolePriv(Integer.parseInt(workspaceId), role, applicationId); +// if(rolePriv != null){ +// priv += rolePriv; +// } +// } +// if (priv == 0){ +// dssMenuSet.remove(subMenu); +// } +// } +// }); +// return new ArrayList<>(dssMenuSet); +// } +//} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspacePrivServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspacePrivServiceImpl.java new file mode 100644 index 000000000..18476226d --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspacePrivServiceImpl.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service.impl; + +import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspacePrivMapper; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspacePrivService; +import org.apache.commons.math3.util.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + + +@Service +public class DSSWorkspacePrivServiceImpl implements DSSWorkspacePrivService { + + + @Autowired + private DSSWorkspacePrivMapper dssWorkspacePrivMapper; + + @Override + @Transactional(rollbackFor = Exception.class) + public void updateRoleMenuPriv(int workspaceId, int menuId, String updater, List> pairs) { + pairs.forEach(pair -> { + int roleId = pair.getKey(); + int priv = pair.getValue() ? 1 : 0; + int count = dssWorkspacePrivMapper.queryCntOfMenuRolePriv(workspaceId, menuId, roleId); + if (count >= 1) { + dssWorkspacePrivMapper.updateRoleMenuPriv(workspaceId, menuId, roleId, priv); + } else { + dssWorkspacePrivMapper.insertMenuRolePriv(workspaceId, menuId, roleId, priv, updater); + } + }); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void updateRoleComponentPriv(int workspaceId, int componentId, String username, List> pairs) { + pairs.forEach(pair -> { + int roleId = pair.getKey(); + int priv = pair.getValue() ? 1 : 0; + int count = dssWorkspacePrivMapper.queryCntOfRCP(workspaceId, componentId, roleId); + if (count >= 1) { + dssWorkspacePrivMapper.updateRoleComponentPriv(workspaceId, componentId, roleId, priv); + } else { + dssWorkspacePrivMapper.insertRolComponentPriv(workspaceId, componentId, roleId, priv, username); + } + }); + } + + @Override + public Integer getRoleId(int workspaceId, String key) { + return dssWorkspacePrivMapper.getRoleId(workspaceId, key); + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceRoleServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceRoleServiceImpl.java new file mode 100644 index 000000000..80ce4dea8 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceRoleServiceImpl.java @@ -0,0 +1,113 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service.impl; + +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSApplicationBean; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceRole; +import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspaceRoleMapper; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleService; +import com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant; +import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Date; +import java.util.List; + + +@Service +public class DSSWorkspaceRoleServiceImpl implements DSSWorkspaceRoleService { + + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSWorkspaceRoleServiceImpl.class); + + @Autowired + private DSSWorkspaceRoleMapper dssWorkspaceRoleMapper; + + @Autowired + private WorkspaceDBHelper workspaceDBHelper; + + @Override + @Transactional(rollbackFor = Exception.class) + public void addWorkspaceRole(String roleName, int workspaceId, List menuIds, List componentIds, String username) { + DSSWorkspaceRole dssRole = new DSSWorkspaceRole(); + dssRole.setWorkspaceId(workspaceId); + dssRole.setFrontName(roleName); + dssRole.setName(roleName); + dssRole.setCreateTime(new Date()); + dssRole.setDescription("workspace{ " + workspaceId + " }添加的role"); + dssWorkspaceRoleMapper.addNewRole(dssRole); + List allMenuIds = workspaceDBHelper.getAllMenuIds(); + if (menuIds.size() > 0) { + dssWorkspaceRoleMapper.updateRoleMenu(dssRole.getId(), workspaceId, menuIds, username, 1); + } + if (menuIds.size() < allMenuIds.size()) { + allMenuIds.removeAll(menuIds); + dssWorkspaceRoleMapper.updateRoleMenu(dssRole.getId(), workspaceId, allMenuIds, username, 0); + } + List allComponentIds = workspaceDBHelper.getAppConnIds(); + if (componentIds.size() > 0) { + dssWorkspaceRoleMapper.updateRoleComponent(dssRole.getId(), workspaceId, componentIds, username, 1); + } + if (componentIds.size() < allComponentIds.size()) { + allComponentIds.removeAll(componentIds); + dssWorkspaceRoleMapper.updateRoleComponent(dssRole.getId(), workspaceId, allComponentIds, username, 0); + } + workspaceDBHelper.retrieveFromDB(); + } + + @Override + public List getRoleInWorkspace(String username, int workspaceId) { + LOGGER.info("get roles for user {} in workspace {}", username, workspaceId); + return dssWorkspaceRoleMapper.getAllRoles(username, workspaceId); + } + + @Override + public Integer getWorkspaceIdByUser(String username) { + List workspaceIds = dssWorkspaceRoleMapper.getWorkspaceIds(username); + Integer defaultWorkspaceId = dssWorkspaceRoleMapper.getDefaultWorkspaceId(DSSWorkspaceConstant.DEFAULT_WORKSPACE_NAME.getValue()); + if (workspaceIds.isEmpty()) { + return defaultWorkspaceId; + } else if (workspaceIds.size() == 1) { + return workspaceIds.get(0); + } else { + workspaceIds.remove(defaultWorkspaceId); + return workspaceIds.get(0); + } + } + + @Override + public int getApiPriv(String username, Integer workspaceId, String roleName, String appName) { + int roleId = dssWorkspaceRoleMapper.getRoleId(roleName, -1); + DSSApplicationBean applicationBean = workspaceDBHelper.getAppConn(appName); + if (applicationBean == null) { + return -1; + } + int appconnId = applicationBean.getId(); + int count = dssWorkspaceRoleMapper.getCount(workspaceId, appconnId, roleId); + if (count >= 1) { + Integer tmpPriv = dssWorkspaceRoleMapper.getPriv(workspaceId, roleId, appconnId); + return tmpPriv != null ? tmpPriv : 0; + } else { + Integer tmpPriv = dssWorkspaceRoleMapper.getPriv(-1, roleId, appconnId); + return tmpPriv != null ? tmpPriv : 0; + } + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceServiceImpl.java new file mode 100644 index 000000000..31a441d5b --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceServiceImpl.java @@ -0,0 +1,668 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service.impl; + +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlySSOAppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.appconn.manager.utils.AppInstanceConstants; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.framework.admin.conf.AdminConf; +import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; +import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkWarnException; +import com.webank.wedatasphere.dss.framework.workspace.bean.*; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceDepartmentVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceFavoriteVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceMenuAppconnVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceMenuVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.*; +import com.webank.wedatasphere.dss.framework.workspace.constant.ApplicationConf; +import com.webank.wedatasphere.dss.framework.workspace.dao.*; +import com.webank.wedatasphere.dss.framework.workspace.exception.DSSWorkspaceDuplicateNameException; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService; +import com.webank.wedatasphere.dss.framework.workspace.service.StaffInfoGetter; +import com.webank.wedatasphere.dss.framework.workspace.util.CommonRoleEnum; +import com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant; +import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; +import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceServerConstant; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.exception.ErrorException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; + +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant.DEFAULT_DEMO_WORKSPACE_NAME; + +//@Service +public class DSSWorkspaceServiceImpl implements DSSWorkspaceService { + private static final Logger LOGGER = LoggerFactory.getLogger(DSSWorkspaceServiceImpl.class); + + @Autowired + private DSSWorkspaceMapper dssWorkspaceMapper; + @Autowired + private DSSWorkspaceUserMapper dssWorkspaceUserMapper; + @Autowired + private DSSWorkspaceMenuMapper dssWorkspaceMenuMapper; + @Autowired + private DSSWorkspaceInfoMapper dssWorkspaceInfoMapper; + @Autowired + private WorkspaceDBHelper workspaceDBHelper; + @Autowired + private DSSWorkspaceService dssWorkspaceService; + @Autowired + private DSSWorkspaceUserService dssWorkspaceUserService; + @Autowired + private DssAdminUserService dssUserService; + @Autowired + private DSSMenuRoleMapper dssMenuRoleMapper; + @Autowired + private DSSWorkspaceHomepageMapper dssWorkspaceHomepageMapper; + @Autowired + private DSSComponentRoleMapper dssComponentRoleMapper; + @Autowired + private WorkspaceMapper workspaceMapper; + + @Autowired + private StaffInfoGetter staffInfoGetter; + + @Autowired + private DSSWorkspaceRoleService dssWorkspaceRoleService; + + //创建工作空间 + @Override + @Transactional(rollbackFor = Exception.class) + public int createWorkspace(String workspaceName, String tags, String userName, + String description, String department, String productName, String workspaceType) throws ErrorException { + DSSWorkspace dssWorkspace = new DSSWorkspace(); + dssWorkspace.setDescription(description); + dssWorkspace.setName(workspaceName); + dssWorkspace.setLabel(tags); + dssWorkspace.setCreateBy(userName); + dssWorkspace.setDepartment(department); + dssWorkspace.setProduct(productName); + dssWorkspace.setCreateTime(new Date()); + dssWorkspace.setLastUpdateTime(new Date()); + dssWorkspace.setLastUpdateUser(userName); + dssWorkspace.setWorkspaceType(workspaceType); + try { + dssWorkspaceMapper.createWorkSpace(dssWorkspace); + } catch (Exception e) { + DSSWorkspaceDuplicateNameException exception1 = new DSSWorkspaceDuplicateNameException(50010, "工作空间名重复"); + exception1.initCause(e); + throw exception1; + } + Long userId = dssWorkspaceUserMapper.getUserID(userName); + dssWorkspaceUserMapper.setUserRoleInWorkspace(dssWorkspace.getId(), + workspaceDBHelper.getRoleIdByName(CommonRoleEnum.ADMIN.getName()), userName, userName, userId); + dssMenuRoleMapper.insertBatch(workspaceDBHelper.generateDefaultWorkspaceMenuRole(dssWorkspace.getId(), userName)); + dssWorkspaceHomepageMapper.insertBatch(workspaceDBHelper.generateDefaultWorkspaceHomepage(dssWorkspace.getId(), userName)); + dssComponentRoleMapper.insertBatch(workspaceDBHelper.generateDefaultWorkspaceComponentPrivs(dssWorkspace.getId(), userName)); + setAllRolesToWorkspaceCreator(dssWorkspace.getId(), userName); + return dssWorkspace.getId(); + } + + private void setAllRolesToWorkspaceCreator(int workspaceId, String userName) { + List roleIds = dssWorkspaceService.getWorkspaceRoles(workspaceId) + .stream() + .map(DSSWorkspaceRoleVO::getRoleId) + .collect(Collectors.toList()); + dssWorkspaceUserService.updateWorkspaceUser(roleIds, workspaceId, userName, userName); + } + + + //把用户及角色添加到工作空间 + @Override + @Transactional(rollbackFor = Exception.class) + public void addWorkspaceUser(List roleIds, Workspace workspace, String userName, String creator, String userId) { + //根据用户名 从用户表拿到用户id +// Long userId = dssUserService.getUserID(userName); + if (userId == null) { + //保存 - dss_user + dssUserService.insertOrUpdateUser(userName, workspace); + } + //保存 - 保存用户角色关系 dss_workspace_user_role + for (Integer roleId : roleIds) { + dssWorkspaceUserMapper.setUserRoleInWorkspace((int) workspace.getWorkspaceId(), roleId, userName, creator, userId == null ? null : Long.parseLong(userId)); + } + } + + //获取所有的工作空间 + @Override + public List getWorkspaces(String userName) { + List workspaces = dssWorkspaceMapper.getWorkspaces(userName); + //用于展示demo的工作空间是不应该返回的,除非用户是管理员 + if (dssWorkspaceUserMapper.isAdmin(userName) == 1) { + return workspaces; + } else { + //踢掉那个演示demo工作空间 + List retWorkspaces = new ArrayList<>(); + String[] defaultDemoWorkspaceNames = DEFAULT_DEMO_WORKSPACE_NAME.getValue().split(","); + for (DSSWorkspace workspace : workspaces) { + if (!ArrayUtils.contains(defaultDemoWorkspaceNames, workspace.getName())) { + retWorkspaces.add(workspace); + } + } + return retWorkspaces; + } + } + + @Override + public DSSWorkspaceHomePageVO getWorkspaceHomePage(String userName, String moduleName) throws DSSErrorException { + //根据用户名 拿到用户ID + //根据用户id 和工作空间id 拿到 角色id + //根据role id 和工作空间id 拿到 重定向的 url + List tempWorkspaceIds = dssWorkspaceUserMapper.getWorkspaceIds(userName); +// if (tempWorkspaceIds == null || tempWorkspaceIds.isEmpty()) { +// throw new DSSErrorException(30020, "该账号尚未加入工作空间,请联系管理员分配工作空间及用户角色"); +// } + List workspaceIds = new ArrayList<>(); + tempWorkspaceIds.stream(). + map(dssWorkspaceInfoMapper::getWorkspaceNameById). + filter(name -> !DEFAULT_DEMO_WORKSPACE_NAME.getValue().equals(name)). + map(dssWorkspaceInfoMapper::getWorkspaceIdByName). + forEach(workspaceIds::add); + DSSWorkspaceHomePageVO dssWorkspaceHomePageVO = new DSSWorkspaceHomePageVO(); + if (workspaceIds.size() == 0) { + Long userId = dssWorkspaceUserMapper.getUserID(userName); + int workspaceId = dssWorkspaceInfoMapper.getWorkspaceIdByName(DSSWorkspaceConstant.DEFAULT_WORKSPACE_NAME.getValue()); + dssWorkspaceUserMapper.setUserRoleInWorkspace(workspaceId, workspaceDBHelper.getRoleIdByName(CommonRoleEnum.ANALYSER.getName()), + userName, "system", userId); + String homepageUrl = dssWorkspaceUserMapper.getHomepageUrl(workspaceId, workspaceDBHelper.getRoleIdByName(CommonRoleEnum.ANALYSER.getName())); + if (ApplicationConf.HOMEPAGE_MODULE_NAME.getValue().equalsIgnoreCase(moduleName)) { + homepageUrl = ApplicationConf.HOMEPAGE_URL.getValue() + workspaceIds.get(0); + } + if (StringUtils.isEmpty(homepageUrl)) { + homepageUrl = "/home" + "?workspaceId=" + workspaceId; + } + dssWorkspaceHomePageVO.setHomePageUrl(homepageUrl); + dssWorkspaceHomePageVO.setWorkspaceId(workspaceId); + dssWorkspaceHomePageVO.setRoleName(CommonRoleEnum.ANALYSER.getName()); + } else if (workspaceIds.size() == 1) { + //只有一个工作空间,那么就返回该工作空间的首页 + List roleIds = dssWorkspaceUserMapper.getRoleInWorkspace(workspaceIds.get(0), userName); + int minRoleId = Collections.min(roleIds); + String homepageUrl = dssWorkspaceUserMapper.getHomepageUrl(workspaceIds.get(0), minRoleId); + if ("/workspace".equals(homepageUrl)) { + homepageUrl = "/workspaceHome"; + } + if (StringUtils.isNotEmpty(homepageUrl)) { + homepageUrl = homepageUrl + "?workspaceId=" + workspaceIds.get(0); + } else { + homepageUrl = "/home" + "?workspaceId=" + workspaceIds.get(0); + } + if (ApplicationConf.HOMEPAGE_MODULE_NAME.getValue().equalsIgnoreCase(moduleName)) { + homepageUrl = ApplicationConf.HOMEPAGE_URL.getValue() + workspaceIds.get(0); + } + dssWorkspaceHomePageVO.setHomePageUrl(homepageUrl); + dssWorkspaceHomePageVO.setWorkspaceId(workspaceIds.get(0)); + dssWorkspaceHomePageVO.setRoleName(workspaceDBHelper.getRoleNameById(minRoleId)); + } else { + //排除掉默认的默认工作空间bdapWorkspace + String homepageUrl = "/workspaceHome?workspaceId=" + workspaceIds.get(0); + if (ApplicationConf.HOMEPAGE_MODULE_NAME.getValue().equalsIgnoreCase(moduleName)) { + homepageUrl = ApplicationConf.HOMEPAGE_URL.getValue() + workspaceIds.get(0); + } + dssWorkspaceHomePageVO.setWorkspaceId(workspaceIds.get(0)); + dssWorkspaceHomePageVO.setHomePageUrl(homepageUrl); + } + return dssWorkspaceHomePageVO; + } + + @Override + public List getWorkspaceUsers(String workspaceId, String department, String username, + String roleName, int pageNow, int pageSize, List total) { + int roleId = -1; + if (StringUtils.isNotEmpty(roleName)) { + roleId = workspaceDBHelper.getRoleIdByName(roleName); + } + PageHelper.startPage(pageNow, pageSize); + List workspaceUsers = new ArrayList<>(); + try { + workspaceUsers = dssWorkspaceUserMapper.getWorkspaceUsers(workspaceId, username); + } finally { + //PageHelper.clearPage(); + } + PageInfo pageInfo = new PageInfo<>(workspaceUsers); + total.add(pageInfo.getTotal()); + List dssWorkspaceUserVOs = new ArrayList<>(); + for (DSSWorkspaceUser workspaceUser : workspaceUsers) { + List roles = dssWorkspaceUserMapper.getRoleInWorkspace(Integer.parseInt(workspaceId), workspaceUser.getUsername()); + dssWorkspaceUserVOs.add(changeToUserVO(workspaceUser, roles)); + } + return dssWorkspaceUserVOs; + } + + private DSSWorkspaceUserVO changeToUserVO(DSSWorkspaceUser dssWorkspaceUser, List roles) { + DSSWorkspaceUserVO vo = new DSSWorkspaceUserVO(); + String userName = dssWorkspaceUser.getUsername(); + vo.setName(userName); + String orgFullName = staffInfoGetter.getFullOrgNameByUsername(userName); + if (StringUtils.isNotEmpty(orgFullName)) { + try { + String departmentName = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[0]; + String officeName = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[1]; + vo.setDepartment(departmentName); + vo.setOffice(officeName); + } catch (Exception e) { + //LOGGER.warn("fail to get department and office {} ", e.getMessage()); + vo.setDepartment(WorkspaceServerConstant.DEFAULT_DEPARTMENT); + vo.setOffice(WorkspaceServerConstant.DEFAULT_OFFICE); + } + } + vo.setRoles(roles); + vo.setCreator(dssWorkspaceUser.getCreator()); + vo.setJoinTime(dssWorkspaceUser.getJoinTime()); + return vo; + } + + @Override + public List getWorkspaceRoles(int workspaceId) { + return workspaceDBHelper.getRoleVOs(workspaceId); + } + + @Override + public DSSWorkspacePrivVO getWorkspaceMenuPrivs(String workspaceId) { + DSSWorkspacePrivVO dssWorkspacePrivVO = new DSSWorkspacePrivVO(); + dssWorkspacePrivVO.setWorkspaceId(Integer.parseInt(workspaceId)); + List workspaceRoleVOList = workspaceDBHelper.getRoleVOs(Integer.parseInt(workspaceId)); + dssWorkspacePrivVO.setRoleVOS(workspaceRoleVOList); + List dssWorkspaceMenuPrivVOList = new ArrayList<>(); + List dssWorkspaceMenuRolePrivList = dssWorkspaceMapper.getDSSWorkspaceMenuPriv(workspaceId); + List defaultWorkspaceMenuRolePrivList = dssWorkspaceMapper.getDefaultWorkspaceMenuPriv(); + for (DSSWorkspaceMenuRolePriv v : defaultWorkspaceMenuRolePrivList) { + if (!dssWorkspaceMenuRolePrivList.contains(v)) { + v.setPriv(0); + dssWorkspaceMenuRolePrivList.add(v); + } + } + Map> map = new HashMap<>(); + for (DSSWorkspaceMenuRolePriv dssWorkspaceMenuPriv : dssWorkspaceMenuRolePrivList) { + int menuId = dssWorkspaceMenuPriv.getMenuId(); + if (!map.containsKey(menuId)) { + map.put(menuId, new ArrayList<>()); + } + map.get(menuId).add(dssWorkspaceMenuPriv); + } + // 得到(menuId, dssWorkspaceMenuRolePrivs) + + map.forEach((k, v) -> { + DSSWorkspaceMenuPrivVO vo = new DSSWorkspaceMenuPrivVO(); + vo.setId(k); + if (workspaceDBHelper.getMenuNameById(k) != null) { + vo.setName(workspaceDBHelper.getMenuNameById(k).getTitleCn()); + Map menuPrivs = new HashMap<>(); + workspaceRoleVOList.forEach(role -> { + int roleId = role.getRoleId(); + boolean isContain = false; + for (DSSWorkspaceMenuRolePriv dssWorkspaceMenuRolePriv : v) { + if (roleId == dssWorkspaceMenuRolePriv.getRoleId()) { + menuPrivs.put(role.getRoleName(), dssWorkspaceMenuRolePriv.getPriv() == 1); + isContain = true; + break; + } + } + if (!isContain) { + menuPrivs.put(role.getRoleName(), false); + } + + }); + vo.setMenuPrivs(menuPrivs); + dssWorkspaceMenuPrivVOList.add(vo); + } + + }); + dssWorkspacePrivVO.setMenuPrivVOS(dssWorkspaceMenuPrivVOList); + + List dssWorkspaceComponentPrivVOList = new ArrayList<>(); + List dssWorkspaceComponentRolePrivList = + dssWorkspaceMenuMapper.getComponentRolePriv(Integer.parseInt(workspaceId)); + List defaultDssWorkspaceComponentRolePrivList = dssWorkspaceMenuMapper.getDefaultComponentRolePriv01(); + for (DSSWorkspaceComponentRolePriv p : defaultDssWorkspaceComponentRolePrivList) { + if (!dssWorkspaceComponentRolePrivList.contains(p)) { + p.setPriv(0); + dssWorkspaceComponentRolePrivList.add(p); + } + } + Map> map1 = new HashMap<>(); + for (DSSWorkspaceComponentRolePriv dssWorkspaceComponentRolePriv : dssWorkspaceComponentRolePrivList) { + Integer componentId = dssWorkspaceComponentRolePriv.getComponentId(); + if (componentId == null) { + continue; + } + if (!map1.containsKey(componentId)) { + List tempList = new ArrayList<>(); + tempList.add(dssWorkspaceComponentRolePriv); + map1.put(componentId, tempList); + } else { + map1.get(componentId).add(dssWorkspaceComponentRolePriv); + } + } + map1.forEach((k, v) -> { + DSSWorkspaceComponentPrivVO vo = new DSSWorkspaceComponentPrivVO(); + vo.setId(k); + Map componentPrivs = new HashMap<>(); + + if (workspaceDBHelper.getAppConn(k) != null) { + vo.setName(workspaceDBHelper.getAppConn(k).getName()); + workspaceRoleVOList.forEach(role -> { + int roleId = role.getRoleId(); + boolean isContain = false; + for (DSSWorkspaceComponentRolePriv dssWorkspaceComponentRolePriv : v) { + if (roleId == dssWorkspaceComponentRolePriv.getRoleId()) { + componentPrivs.put(role.getRoleName(), dssWorkspaceComponentRolePriv.getPriv() != null && dssWorkspaceComponentRolePriv.getPriv() == 1); + isContain = true; + break; + } + } + if (!isContain) { + componentPrivs.put(role.getRoleName(), false); + } + + }); + vo.setComponentPrivs(componentPrivs); + dssWorkspaceComponentPrivVOList.add(vo); + + } + + }); + dssWorkspacePrivVO.setComponentPrivVOS(dssWorkspaceComponentPrivVOList); + return dssWorkspacePrivVO; + } + + @Override + public DSSWorkspaceOverviewVO getOverview(String username, int workspaceId, boolean isEnglish) { + DSSWorkspaceOverviewVO dssWorkspaceOverviewVO = new DSSWorkspaceOverviewVO(); + DSSWorkspace dssWorkspace = dssWorkspaceInfoMapper.getWorkspace(workspaceId); + dssWorkspaceOverviewVO.setTitle(dssWorkspace.getName()); + dssWorkspaceOverviewVO.setDescription(dssWorkspace.getDescription()); + return dssWorkspaceOverviewVO; + } + + @Override + public DSSWorkspaceHomepageSettingVO getWorkspaceHomepageSettings(int workspaceId) { + DSSWorkspaceHomepageSettingVO dssWorkspaceHomepageSettingVO = new DSSWorkspaceHomepageSettingVO(); + + List dssWorkspaceHomepageSettings = dssWorkspaceMenuMapper.getWorkspaceHomepageSettings(workspaceId); + List roleHomepageList = new ArrayList<>(); + dssWorkspaceHomepageSettings.forEach(homepage -> { + DSSWorkspaceHomepageSettingVO.RoleHomepage roleHomepage = new DSSWorkspaceHomepageSettingVO.RoleHomepage(); + roleHomepage.setHomepageName(workspaceDBHelper.getHomepageName(homepage.getHomepageUrl())); + roleHomepage.setHomepageUrl(homepage.getHomepageUrl()); + roleHomepage.setRoleId(homepage.getRoleId()); + roleHomepage.setRoleName(workspaceDBHelper.getRoleNameById(homepage.getRoleId())); + roleHomepage.setRoleFrontName(workspaceDBHelper.getRoleFrontName(homepage.getRoleId())); + roleHomepageList.add(roleHomepage); + }); + dssWorkspaceHomepageSettingVO.setRoleHomepages(roleHomepageList); + return dssWorkspaceHomepageSettingVO; + } + + @Override + public String getWorkspaceName(String workspaceId) { + return dssWorkspaceInfoMapper.getWorkspaceNameById(Integer.parseInt(workspaceId)); + } + + @Override + public boolean checkAdmin(String userName) { + return dssWorkspaceUserMapper.isAdmin(userName) == 1; + } + + @Override + public boolean checkAdminByWorkspace(String username, int workspaceId) { + List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, workspaceId); + return roles.stream().anyMatch(role -> role.equalsIgnoreCase("admin")); + } + + @Override + public List getDepartments() { + List allDepartments = staffInfoGetter.getAllDepartments(); + List departmentVOs = new ArrayList<>(); + int count = 1; + for (String department : allDepartments) { + DepartmentVO departmentVO = new DepartmentVO(); + departmentVO.setFrontName(department); + departmentVO.setName(department); + departmentVO.setId(count); + departmentVOs.add(departmentVO); + count++; + } + return departmentVOs; + } + + @Override + public List getWorkspaceUsersByRole(int workspaceId, String roleName, List totals, int pageNow, int pageSize) { + int roleId = workspaceDBHelper.getRoleIdByName(roleName); + PageHelper.startPage(pageNow, pageSize); + List workspaceUsers = new ArrayList<>(); + try { + workspaceUsers = dssWorkspaceUserMapper.getWorkspaceUsersByRole(workspaceId, roleId); + } finally { + PageHelper.clearPage(); + } + PageInfo pageInfo = new PageInfo<>(workspaceUsers); + totals.add(pageInfo.getTotal()); + List dssWorkspaceUserVOs = new ArrayList<>(); + for (DSSWorkspaceUser workspaceUser : workspaceUsers) { + List roles = dssWorkspaceUserMapper.getRoleInWorkspace(workspaceId, workspaceUser.getUsername()); + dssWorkspaceUserVOs.add(changeToUserVO(workspaceUser, roles)); + } + return dssWorkspaceUserVOs; + } + + private DSSWorkspace getWorkspace(Supplier workspaceSupplier, String username) throws DSSErrorException { + DSSWorkspace dssWorkSpace = workspaceSupplier.get(); + if (dssWorkSpace == null) { + throw new DSSErrorException(30022, "workspace is not exists."); + } + List users = dssWorkspaceUserMapper.getAllWorkspaceUsers(dssWorkSpace.getId()); + if (!users.contains(username)) { + throw new DSSErrorException(30021, "You have no permission to access this workspace " + dssWorkSpace.getName()); + } + String originDepartId = dssWorkSpace.getDepartment(); + if (StringUtils.isNotBlank(originDepartId)) { + String departName = workspaceMapper.getDepartName(Long.valueOf(originDepartId)); + dssWorkSpace.setDepartment(departName); + } + return dssWorkSpace; + } + + @Override + public DSSWorkspace getWorkspacesById(Long id, String username) throws DSSErrorException { + return getWorkspace(() -> workspaceMapper.getWorkspaceById(id), username); + } + + @Override + public DSSWorkspace getWorkspacesByName(String workspaceName, String username) throws DSSErrorException { + return getWorkspace(() -> { + List dssWorkspaces = workspaceMapper.findByWorkspaceName(workspaceName); + if (dssWorkspaces == null || dssWorkspaces.isEmpty()) { + return null; + } else if (dssWorkspaces.size() > 1) { + throw new DSSFrameworkWarnException(30021, "Too many workspaces named " + workspaceName); + } else { + return dssWorkspaces.get(0); + } + }, username); + } + + @Override + public Long addWorkspace(String userName, String name, String department, String label, String description) { + DSSWorkspace dssWorkspace = new DSSWorkspace(); + dssWorkspace.setName(name); + dssWorkspace.setDepartment(department); + dssWorkspace.setDescription(description); + dssWorkspace.setLabel(label); + dssWorkspace.setCreateBy(userName); + dssWorkspace.setSource("create by user"); + dssWorkspace.setLastUpdateUser(userName); + workspaceMapper.addWorkSpace(dssWorkspace); + return (long) dssWorkspace.getId(); + } + + @Override + public boolean existWorkspaceName(String name) { + return !workspaceMapper.findByWorkspaceName(name).isEmpty(); + } + + @Override + public List getWorkSpaceDepartments() { + // TODO: service层和dao层完善 + WorkspaceDepartmentVo dp = new WorkspaceDepartmentVo(); + dp.setId(1L); + dp.setName("应用开发组"); + WorkspaceDepartmentVo di = new WorkspaceDepartmentVo(); + di.setId(2L); + di.setName("平台研发组"); + List departments = new ArrayList<>(); + departments.add(dp); + departments.add(di); + return departments; + } + + private List getMenuAppInstances(List menuVos, List userMenuAppConnIds, + DSSWorkspace dssworkspace, Workspace workspace, + boolean isChinese) { + for (WorkspaceMenuVo menuVo : menuVos) { + Long menuId = menuVo.getId(); + List menuAppconns = isChinese ? workspaceMapper.getMenuAppInstancesCn(menuId) : workspaceMapper.getMenuAppInstancesEn(menuId); + for (WorkspaceMenuAppconnVo menuAppconn : menuAppconns) { + // 如果该工作空间中用户拥有该组件权限,则该组件的accessable属性为true;否则为false + menuAppconn.setAccessable(userMenuAppConnIds.contains(menuAppconn.getId())); + AppConn appConn = AppConnManager.getAppConnManager().getAppConn(menuAppconn.getName()); + List instanceList = new ArrayList<>(); + SSOUrlBuilderOperation operation; + if(appConn instanceof OnlySSOAppConn) { + operation = ((OnlySSOAppConn) appConn).getOrCreateSSOStandard().getSSOBuilderService().createSSOUrlBuilderOperation(); + SSOHelper.setSSOUrlBuilderOperation(operation, workspace); + operation.setAppName(appConn.getAppDesc().getAppName()); + } else { + operation = null; + } + appConn.getAppDesc().getAppInstances().forEach(appInstance -> { + String label = String.join(",", appInstance.getLabels().stream() + .map(l -> ((EnvDSSLabel) l).getEnv()).toArray(String[]::new)); + String selectedName = getAppInstanceTitle(appConn, appInstance, isChinese); + String homepageUri = AppInstanceConstants.getHomepageUrl(appInstance, operation, (long) dssworkspace.getId(), dssworkspace.getName()); + instanceList.add(new DSSApplicationBean(selectedName, appInstance.getBaseUrl(), + homepageUri, label)); + }); + menuAppconn.setAppInstances(instanceList); + } + menuVo.setAppconns(menuAppconns); + } + return menuVos; + } + + protected String getAppInstanceTitle(AppConn appConn, AppInstance appInstance, boolean isChinese) { + if (isChinese) { + return "进入 " + appConn.getAppDesc().getAppName(); + } else { + return "Enter " + appConn.getAppDesc().getAppName(); + } + } + + @Override + public List getWorkspaceAppConns(Workspace workspace, Long workspaceId, String username, + boolean isChinese) throws DSSErrorException { + DSSWorkspace dssWorkspace = getWorkspacesById(workspaceId, username); + List appconnMenuVos = isChinese ? workspaceMapper.getAppConnMenuCn() : workspaceMapper.getAppConnMenuEn(); + List userMenuAppConnIds = dssWorkspaceMapper.getUserMenuAppConnId(username, workspaceId); + return getMenuAppInstances(appconnMenuVos, userMenuAppConnIds, dssWorkspace, workspace, isChinese); + } + + @Override + public List getWorkspaceFavorites(Long workspaceId, String username, boolean isChinese, String type) { + checkScriptis(workspaceId, username, "dingyiding"); + return isChinese ? workspaceMapper.getWorkspaceFavoritesCn(username, workspaceId, type) : workspaceMapper.getWorkspaceFavoritesEn(username, workspaceId, type); + } + + /** + * 保证Scriptis入口一直在页面上方 + * + * @param workspaceId + * @param username + * @param type + */ + private void checkScriptis(Long workspaceId, String username, String type) { + Long scriptisMenuAppId = dssWorkspaceMapper.getMenuAppIdByName("Scriptis"); + int exists = dssWorkspaceMapper.getByMenuAppIdAndUser(scriptisMenuAppId, workspaceId, username, type); + if (exists < 1) { + addFavorite(username, workspaceId, scriptisMenuAppId, type); + } + } + + @Override + public Long addFavorite(String username, Long workspaceId, Long menuApplicationId, String type) { + DSSFavorite dssFavorite = new DSSFavorite(); + dssFavorite.setUsername(username); + dssFavorite.setWorkspaceId(workspaceId); + dssFavorite.setMenuAppConnId(menuApplicationId); + // todo: order will from the front end + dssFavorite.setOrder(1); + dssFavorite.setCreateBy(username); + dssFavorite.setLastUpdateUser(username); + dssFavorite.setType(type); + workspaceMapper.addFavorite(dssFavorite); + return dssFavorite.getId(); + } + + @Override + public Long deleteFavorite(String username, Long applicationId, Long workspaceId, String type) { + workspaceMapper.deleteFavorite(username, applicationId, workspaceId, type); + return applicationId; + } + + + /** + * 是否超级管理员 + * + * @param workspaceId + * @param username + * @return + */ + @Override + public boolean isAdminUser(Long workspaceId, String username) { + DSSWorkspace workspace = workspaceMapper.getWorkspaceById(workspaceId); + List roles = dssWorkspaceUserMapper.getRoleInWorkspace(workspaceId.intValue(), username); + if (roles != null && roles.size() > 0) { + for (Integer role : roles) { + if (role == 1) { + return true; + } + } + } + //默认空间配置的超级管理员,返回true + return (workspace.getName().equals(DSSWorkspaceConstant.DEFAULT_WORKSPACE_NAME.getValue()) && + org.apache.commons.lang3.ArrayUtils.contains(AdminConf.SUPER_ADMIN_LIST, username)) || + username.equals(workspace.getCreateBy()); + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceUserServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceUserServiceImpl.java new file mode 100644 index 000000000..fdbf93400 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceUserServiceImpl.java @@ -0,0 +1,113 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service.impl; + +import com.webank.wedatasphere.dss.framework.workspace.bean.StaffInfo; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.StaffInfoVO; +import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspaceUserMapper; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService; +import com.webank.wedatasphere.dss.framework.workspace.service.StaffInfoGetter; +import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceServerConstant; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.stream.Collectors; + + +@Service +public class DSSWorkspaceUserServiceImpl implements DSSWorkspaceUserService { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSWorkspaceUserServiceImpl.class); + @Autowired + private DSSWorkspaceUserMapper dssWorkspaceUserMapper; + + @Autowired + private StaffInfoGetter staffInfoGetter; + + @Override + @Transactional(rollbackFor = Throwable.class) + public void updateWorkspaceUser(List roles, int workspaceId, String userName, String creator) { + dssWorkspaceUserMapper.removeAllRolesForUser(userName, workspaceId); + roles.forEach(role ->{ + dssWorkspaceUserMapper.setUserRoleInWorkspace(workspaceId, role, userName, creator, 0L); + }); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteWorkspaceUser(String userName, int workspaceId) { + dssWorkspaceUserMapper.removeAllRolesForUser(userName, workspaceId); + } + + @Override + public List listAllDSSUsers() { + //需要将esb带来的所有用户进行返回 + List staffInfos = staffInfoGetter.getAllUsers(); + return staffInfos.stream().map(this::staffToDSSUser).collect(Collectors.toList()); + } + + private StaffInfoVO staffToDSSUser(StaffInfo staffInfo){ + StaffInfoVO staffInfoVO = new StaffInfoVO(); + String orgFullName = staffInfo.getOrgFullName(); + if (StringUtils.isNotEmpty(orgFullName)){ + try{ + String departmentName = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[0]; + String officeName = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[1]; + staffInfoVO.setDepartment(departmentName); + staffInfoVO.setOffice(officeName); + }catch(Exception e){ + //LOGGER.warn("fail to get department and office {} ", e.getMessage()); + staffInfoVO.setDepartment(WorkspaceServerConstant.DEFAULT_DEPARTMENT); + staffInfoVO.setOffice(WorkspaceServerConstant.DEFAULT_OFFICE); + } + } + staffInfoVO.setUsername(staffInfo.getEnglishName()); + staffInfoVO.setId(staffInfo.getId()); + return staffInfoVO; + } + + @Override + public List getAllWorkspaceUsers(int workspaceId) { + return dssWorkspaceUserMapper.getAllWorkspaceUsers(workspaceId); + } + + @Override + public List getUserWorkspaceIds(String userName) { + List tempWorkspaceIds = dssWorkspaceUserMapper.getWorkspaceIds(userName); + return tempWorkspaceIds; + } + + @Override + public List getWorkspaceEditUsers(int workspaceId) { + return dssWorkspaceUserMapper.getWorkspaceEditUsers(workspaceId); + } + + @Override + public List getWorkspaceReleaseUsers(int workspaceId) { + return dssWorkspaceUserMapper.getWorkspaceReleaseUsers(workspaceId); + } + + @Override + public Long getCountByUsername(String username,int workspaceId){ + return dssWorkspaceUserMapper.getCountByUsername(username,workspaceId); + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DefaultStaffInfoGetter.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DefaultStaffInfoGetter.java new file mode 100644 index 000000000..da752f399 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DefaultStaffInfoGetter.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.service.impl; + +import com.google.common.collect.Lists; +import com.webank.wedatasphere.dss.framework.workspace.bean.StaffInfo; +import com.webank.wedatasphere.dss.framework.workspace.service.StaffInfoGetter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.stereotype.Component; + +import java.util.Arrays; +import java.util.List; + + + +public class DefaultStaffInfoGetter implements StaffInfoGetter { + + @Override + public List getAllUsers() { + return Lists.newArrayList(new StaffInfo("1","hadoop", "WeDataSphere")); + } + + @Override + public String getFullOrgNameByUsername(String username) { + return "WeDataSphere"; + } + + @Override + public List getAllDepartments() { + List allDepartments = Arrays.asList("WeDataSphere","DataSP", "linkis"); + return allDepartments; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/ApplicationUtils.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/ApplicationUtils.java new file mode 100644 index 000000000..2ab47b549 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/ApplicationUtils.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.util; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + + +public class ApplicationUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationUtils.class); + + private static final String REDIRECT_FORMAT = "%s?redirect=%s&dssurl=${dssurl}&cookies=${cookies}"; + + private static String URLEndoder(String str){ + try { + return URLEncoder.encode(str,"utf-8"); + } catch (UnsupportedEncodingException e) { + LOGGER.warn("endoe failed:",e); + return str; + } + } + + public static String redirectUrlFormat(String redirectUrl, String url){ + return String.format(REDIRECT_FORMAT,redirectUrl,URLEndoder(url)); + } + + public static void main(String[] args) throws DSSErrorException { + System.out.println(redirectUrlFormat("http://127.0..0.1:8090/qualitis/api/v1/redirect","http://127.0..0.1:8090/#/projects/list?id={projectId}&flow=true")); + } + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonAppConnEnum.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonAppConnEnum.java new file mode 100644 index 000000000..d1057507d --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonAppConnEnum.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.framework.workspace.util; + +public enum CommonAppConnEnum { + /** + * 工作空间默认组件 + * 需要和dss_appconn插入的name一致 + */ + WORKFLOW("workflow"), + SCRIPTIS("scriptis"); + + private String name; + + CommonAppConnEnum(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonMenuEnum.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonMenuEnum.java new file mode 100644 index 000000000..74d10d1af --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonMenuEnum.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.framework.workspace.util; + +public enum CommonMenuEnum { + /** + * 中文名需要和数据库插入的一致 + */ + APPLICATION_DEVELOPMENT("应用开发"), + DATA_ANALYSIS("数据分析"), + PRODUCTION_OPERATION("生产运维"), + DATA_QUALITY("数据质量"), + DATA_EXCHANGE("数据交换"), + DATA_APPLICATION("数据应用"), + ADMIN_FUNCTOIN("管理员功能"); + + CommonMenuEnum(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + private String name; + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonRoleEnum.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonRoleEnum.java new file mode 100644 index 000000000..507ba7cfa --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/CommonRoleEnum.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.util; + + +public enum CommonRoleEnum { + /** + * 通用角色的枚举,name必须和数据库一致 + */ + ADMIN(1, "admin", "管理员"), + MAINTENANCE(2, "maintenance", "运维人员"), + DEVELOPER(3, "developer", "开发人员"), + ANALYSER(4, "analyser", "分析用户"), + OPERATOR(5, "operator", "运营用户"), + BOSS(6, "boss", "领导"), + APIUSER(7, "apiUser", "数据服务用户"); + + private int id; + private String name; + private String frontName; + + private CommonRoleEnum(int id, String name, String frontName) { + this.id = id; + this.name = name; + this.frontName = frontName; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFrontName() { + return frontName; + } + + public void setFrontName(String frontName) { + this.frontName = frontName; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/DSSDictionaryConstant.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/DSSDictionaryConstant.java new file mode 100644 index 000000000..ccac16879 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/DSSDictionaryConstant.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.util; + + +public class DSSDictionaryConstant { + public static final String W_WORKSPACE_DEPARTMENT = "w_workspace_department"; +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/DSSWorkspaceConstant.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/DSSWorkspaceConstant.java new file mode 100644 index 000000000..fd1e269af --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/DSSWorkspaceConstant.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.util; + +import org.apache.linkis.common.conf.CommonVars; + + +public interface DSSWorkspaceConstant { + String WORKSPACE_ID_STR = "workspaceId"; + + CommonVars DEFAULT_WORKSPACE_NAME + = CommonVars.apply("wds.dss.workspace.default.name", "bdapWorkspace"); + + CommonVars DEFAULT_DEMO_WORKSPACE_NAME = CommonVars.apply("wds.dss.default.demo.workspace", "WDS_DSS_DEMO"); + + String WORKSPACE_MANAGEMENT_NAME = CommonVars.apply("wds.dss.workspace.management.name", "工作空间管理").getValue(); + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/RestfulUtils.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/RestfulUtils.java new file mode 100644 index 000000000..d5c97c15c --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/RestfulUtils.java @@ -0,0 +1,45 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + + package com.webank.wedatasphere.dss.framework.workspace.util; + + @Deprecated + public class RestfulUtils { + +// public static Response dealError(String reason){ +// Message message = Message.error(reason); +// +// return Message.response(message); +// } +// +// public static Response dealOk(String msg){ +// Message message = Message.ok(msg); +// return Message.messageToResponse(message); +// } +// +// +// +// @SafeVarargs +// public static Response dealOk(String msg, Pair... data){ +// Message message = Message.ok(msg); +// Arrays.stream(data).forEach(p -> message.data(p.getKey(), p.getValue())); +// return Message.messageToResponse(message); +// } + + + } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/RestulHelper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/RestulHelper.java new file mode 100644 index 000000000..286e3f9a6 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/RestulHelper.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.util; + + +import com.webank.wedatasphere.dss.framework.workspace.exception.DSSWorkspaceLoginFailException; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.security.SecurityFilter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; + +@Deprecated +@Component +public class RestulHelper { + + @Autowired + private WorkspaceDBHelper workspaceDBHelper; + + public String getLoginUser(HttpServletRequest request)throws ErrorException { + try{ + return SecurityFilter.getLoginUsername(request); + }catch(Exception e){ + throw new DSSWorkspaceLoginFailException(80013, "You are not logged in"); + } + } + + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceDBHelper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceDBHelper.java new file mode 100644 index 000000000..c90d6246d --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceDBHelper.java @@ -0,0 +1,333 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.util; + + +import com.webank.wedatasphere.dss.appconn.manager.utils.AppInstanceConstants; +import com.webank.wedatasphere.dss.framework.workspace.bean.*; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceRoleVO; +import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspaceRoleMapper; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Component +public class WorkspaceDBHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(WorkspaceDBHelper.class); + + @Autowired + private DSSWorkspaceRoleMapper dssWorkspaceRoleMapper; + + private List dssRoles; + + private final Object lock = new Object(); + + private List dssWorkspaceMenus; + + private List dssApplicationBeans; + + private Map dssHomepageNameMap = new HashMap<>(); + + + @PostConstruct + public void init() { + Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + synchronized (lock) { + dssRoles = dssWorkspaceRoleMapper.getRoles(); + } + } + }, 0, 1, TimeUnit.MINUTES); + + Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + synchronized (lock) { + dssWorkspaceMenus = dssWorkspaceRoleMapper.getWorkspaceMenus(); + } + } + }, 0, 1, TimeUnit.MINUTES); + + Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + synchronized (lock) { + dssApplicationBeans = dssWorkspaceRoleMapper.getDSSAppConns(); + } + } + }, 0, 1, TimeUnit.MINUTES); + + + Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + synchronized (lock) { + String defaultUrl = "/workspace"; + String maintenanceUrl = "/workspaceHome/scheduleCenter"; + String developerUrl = + dssApplicationBeans.stream(). + filter(dssApplicationBean -> "workflow".equals(dssApplicationBean.getName().toLowerCase())). + findFirst().map(WorkspaceDBHelper.this::getHomepageUrl).orElse(defaultUrl); + String analyserUrl = dssApplicationBeans.stream(). + filter(dssApplicationBean -> "linkis".equals(dssApplicationBean.getName().toLowerCase())). + findFirst().map(WorkspaceDBHelper.this::getHomepageUrl).orElse(defaultUrl); + dssHomepageNameMap.put("/workspace", "工作空间首页"); + dssHomepageNameMap.put(developerUrl, "工作流开发"); + dssHomepageNameMap.put(analyserUrl, "意书(Scriptis)"); + dssHomepageNameMap.put(maintenanceUrl, "调度中心首页"); + } + } + }, 1, 1, TimeUnit.MINUTES); + } + + private String getHomepageUrl(DSSApplicationBean bean) { + return AppInstanceConstants.getHomepageUrl(bean.getUrl(), bean.getHomepageUri(), null, null); + } + + public void retrieveFromDB() { + synchronized (lock) { + dssApplicationBeans = dssWorkspaceRoleMapper.getDSSAppConns(); + dssWorkspaceMenus = dssWorkspaceRoleMapper.getWorkspaceMenus(); + dssRoles = dssWorkspaceRoleMapper.getRoles(); + } + } + + public List generateDefaultWorkspaceMenuRole(int workspaceId, String username) { + List list = new ArrayList<>(); + Date date = new Date(System.currentTimeMillis()); + //todo 需要确定各角色默认的菜单权限 + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.DATA_ANALYSIS.getName()), + getRoleIdByName(CommonRoleEnum.ADMIN.getName()), 1, date, username)); + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.PRODUCTION_OPERATION.getName()), + getRoleIdByName(CommonRoleEnum.ADMIN.getName()), 1, date, username)); + + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.DATA_ANALYSIS.getName()), + getRoleIdByName(CommonRoleEnum.MAINTENANCE.getName()), 0, date, username)); + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.PRODUCTION_OPERATION.getName()), + getRoleIdByName(CommonRoleEnum.MAINTENANCE.getName()), 0, date, username)); + + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.DATA_ANALYSIS.getName()), + getRoleIdByName(CommonRoleEnum.DEVELOPER.getName()), 1, date, username)); + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.PRODUCTION_OPERATION.getName()), + getRoleIdByName(CommonRoleEnum.DEVELOPER.getName()), 0, date, username)); + + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.DATA_ANALYSIS.getName()), + getRoleIdByName(CommonRoleEnum.ANALYSER.getName()), 0, date, username)); + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.PRODUCTION_OPERATION.getName()), + getRoleIdByName(CommonRoleEnum.ANALYSER.getName()), 1, date, username)); + + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.DATA_ANALYSIS.getName()), + getRoleIdByName(CommonRoleEnum.OPERATOR.getName()), 0, date, username)); + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.PRODUCTION_OPERATION.getName()), + getRoleIdByName(CommonRoleEnum.OPERATOR.getName()), 0, date, username)); + + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.DATA_ANALYSIS.getName()), + getRoleIdByName(CommonRoleEnum.BOSS.getName()), 1, date, username)); + list.add(new DSSWorkspaceMenuRole(workspaceId, getMenuIdByName(CommonMenuEnum.PRODUCTION_OPERATION.getName()), + getRoleIdByName(CommonRoleEnum.BOSS.getName()), 1, date, username)); + +// if (StringUtils.isNotBlank(ApplicationConf.ESB_APPID)) { +// list.add(new DSSWorkspaceMenuRole(workspaceId, 1, 1, 1, date, username)); +// list.add(new DSSWorkspaceMenuRole(workspaceId, 1, 2, 1, date, username)); +// list.add(new DSSWorkspaceMenuRole(workspaceId, 1, 3, 0, date, username)); +// list.add(new DSSWorkspaceMenuRole(workspaceId, 1, 4, 0, date, username)); +// list.add(new DSSWorkspaceMenuRole(workspaceId, 1, 5, 0, date, username)); +// list.add(new DSSWorkspaceMenuRole(workspaceId, 1, 6, 1, date, username)); +// } + return list; + } + + + public List generateDefaultWorkspaceHomepage(int workspaceId, String username) { + String defaultUrl = "/workspace"; + String maintenanceUrl = "/workspaceHome/scheduleCenter"; + String developerUrl = + dssApplicationBeans.stream(). + filter(dssApplicationBean -> "workflow".equals(dssApplicationBean.getName().toLowerCase())). + findFirst().map(this::getHomepageUrl).orElse(defaultUrl); + String analyserUrl = dssApplicationBeans.stream(). + filter(dssApplicationBean -> "linkis".equals(dssApplicationBean.getName().toLowerCase())). + findFirst().map(this::getHomepageUrl).orElse(defaultUrl); + List dssWorkspaceHomepages = new ArrayList<>(); + Date date = new Date(System.currentTimeMillis()); + dssWorkspaceHomepages.add(new DSSWorkspaceHomepage(workspaceId, getRoleIdByName(CommonRoleEnum.ADMIN.getName()), defaultUrl, date)); + dssWorkspaceHomepages.add(new DSSWorkspaceHomepage(workspaceId, getRoleIdByName(CommonRoleEnum.MAINTENANCE.getName()), maintenanceUrl, date)); + dssWorkspaceHomepages.add(new DSSWorkspaceHomepage(workspaceId, getRoleIdByName(CommonRoleEnum.DEVELOPER.getName()), developerUrl, date)); + dssWorkspaceHomepages.add(new DSSWorkspaceHomepage(workspaceId, getRoleIdByName(CommonRoleEnum.ANALYSER.getName()), analyserUrl, date)); + dssWorkspaceHomepages.add(new DSSWorkspaceHomepage(workspaceId, getRoleIdByName(CommonRoleEnum.OPERATOR.getName()), defaultUrl, date)); + dssWorkspaceHomepages.add(new DSSWorkspaceHomepage(workspaceId, getRoleIdByName(CommonRoleEnum.BOSS.getName()), defaultUrl, date)); + return dssWorkspaceHomepages; + } + + + public List generateDefaultWorkspaceComponentPrivs(int workspaceId, + String username) { + List dssWorkspaceComponentPrivs = new ArrayList<>(); + Date updateTime = new Date(System.currentTimeMillis()); + //admin 添加所有appconn组件的访问权限 + new HashSet<>(getAppConnIds()).forEach(id -> { + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.ADMIN.getName()), + id, 1, updateTime, username)); + }); + + //运维 + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.MAINTENANCE.getName()), + getAppConnIdByName(CommonAppConnEnum.SCRIPTIS.getName()), 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.MAINTENANCE.getName()), + getAppConnIdByName(CommonAppConnEnum.WORKFLOW.getName()), 1, updateTime, username)); + //开发人员 + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.DEVELOPER.getName()), + getAppConnIdByName(CommonAppConnEnum.SCRIPTIS.getName()), 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.DEVELOPER.getName()), + getAppConnIdByName(CommonAppConnEnum.WORKFLOW.getName()), 1, updateTime, username)); + //分析人员 + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.ANALYSER.getName()), + getAppConnIdByName(CommonAppConnEnum.SCRIPTIS.getName()), 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.ANALYSER.getName()), + getAppConnIdByName(CommonAppConnEnum.WORKFLOW.getName()), 1, updateTime, username)); + //运营人员 + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.OPERATOR.getName()), + getAppConnIdByName(CommonAppConnEnum.SCRIPTIS.getName()), 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.OPERATOR.getName()), + getAppConnIdByName(CommonAppConnEnum.WORKFLOW.getName()), 1, updateTime, username)); + //领导 + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.BOSS.getName()), + getAppConnIdByName(CommonAppConnEnum.SCRIPTIS.getName()), 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, getRoleIdByName(CommonRoleEnum.BOSS.getName()), + getAppConnIdByName(CommonAppConnEnum.WORKFLOW.getName()), 1, updateTime, username)); + + /* if(StringUtils.isNotBlank(ApplicationConf.ESB_APPID)){ + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, 1, 3, 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, 2, 3, 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, 3, 3, 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, 4, 3, 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, 5, 3, 1, updateTime, username)); + dssWorkspaceComponentPrivs.add(new DSSWorkspaceComponentPriv(workspaceId, 6, 3, 1, updateTime, username)); + }*/ + return dssWorkspaceComponentPrivs; + } + + + public List getRoles() { + return this.dssRoles; + } + + + public List getRoles(int workspaceId) { + return this.dssRoles.stream(). + filter(dssRole -> dssRole.getWorkspaceId() == workspaceId || dssRole.getWorkspaceId() == -1). + collect(Collectors.toList()); + } + + + public String getRoleFrontName(int roleId) { + DSSWorkspaceRole role = dssRoles.stream().filter(dssRole -> dssRole.getId() == roleId).findFirst().orElse(null); + if (role != null) { + return role.getFrontName(); + } else { + return null; + } + } + + public int getRoleIdByName(String roleName) { + if (StringUtils.isBlank(roleName)) { + return -1; + } + return dssRoles.stream(). + filter(dssRole -> roleName.equals(dssRole.getName())). + findFirst(). + map(DSSWorkspaceRole::getId). + orElse(-1); + } + + + public List getRoleVOs(int workspaceId) { + return getRoles(workspaceId).stream().map(this::changeToRoleVO).collect(Collectors.toList()); + } + + private DSSWorkspaceRoleVO changeToRoleVO(DSSWorkspaceRole dssRole) { + DSSWorkspaceRoleVO vo = new DSSWorkspaceRoleVO(); + vo.setRoleFrontName(dssRole.getFrontName()); + vo.setRoleId(dssRole.getId()); + vo.setRoleName(dssRole.getName()); + return vo; + } + + public DSSWorkspaceMenu getMenuNameById(int menuId) { + if (dssWorkspaceMenus.stream().anyMatch(dssMenu -> dssMenu.getId() == menuId)) { + return dssWorkspaceMenus.stream().filter(dssMenu -> dssMenu.getId() == menuId).findFirst().get(); + } else { + return null; + } + } + + public int getMenuIdByName(String name) { + if (dssWorkspaceMenus.stream().anyMatch(dssMenu -> dssMenu.getName().equals(name))) { + return dssWorkspaceMenus.stream().filter(dssMenu -> dssMenu.getName().equals(name)).findFirst().get().getId(); + } else { + return -1; + } + } + + + public String getRoleNameById(int roleId) { + return dssRoles.stream().filter(dssRole -> dssRole.getId() == roleId).findFirst().orElse(new DSSWorkspaceRole()).getName(); + } + + public DSSApplicationBean getAppConn(int appConnId) { + return dssApplicationBeans.stream(). + filter(dssApplicationBean -> dssApplicationBean.getId() == appConnId). + findFirst(). + orElse(null); + } + + public DSSApplicationBean getAppConn(String appConnName) { + return dssApplicationBeans.stream(). + filter(dssApplicationBean -> dssApplicationBean.getName().equalsIgnoreCase(appConnName)). + findFirst(). + orElse(null); + } + + public Integer getAppConnIdByName(String appConnName) { + return dssApplicationBeans.stream(). + filter(l -> l.getName().equalsIgnoreCase(appConnName)). + findFirst().get().getId(); + } + + public List getAppConnIds() { + return dssApplicationBeans.stream().map(DSSApplicationBean::getId).collect(Collectors.toList()); + } + + public String getHomepageName(String homepage) { + return this.dssHomepageNameMap.get(homepage); + } + + public List getAllMenuIds() { + return dssWorkspaceMenus.stream().map(DSSWorkspaceMenu::getId).collect(Collectors.toList()); + } + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceServerConstant.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceServerConstant.java new file mode 100644 index 000000000..ab75cda43 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceServerConstant.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.util; + + +public interface WorkspaceServerConstant { + String DEFAULT_DEPARTMENT = "基础科技部"; + String DEFAULT_OFFICE = "大数据平台室"; + String DEFAULT_STAFF_SPLIT = "-"; +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceSpringConfig.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceSpringConfig.java new file mode 100644 index 000000000..4924e032b --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceSpringConfig.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.workspace.util; + +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.framework.workspace.service.StaffInfoGetter; +import com.webank.wedatasphere.dss.framework.workspace.service.impl.DSSWorkspaceServiceImpl; +import com.webank.wedatasphere.dss.framework.workspace.service.impl.DefaultStaffInfoGetter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class WorkspaceSpringConfig { + + @Bean + @ConditionalOnMissingBean + public StaffInfoGetter getStaffInfoGetter() { + return new DefaultStaffInfoGetter(); + } + + @Bean + @ConditionalOnMissingBean + public DSSWorkspaceService createDSSWorkspaceService() { + return new DSSWorkspaceServiceImpl(); + } +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/pom.xml b/dss-framework/framework-plugins/dss-framework-migrate-server/pom.xml new file mode 100644 index 000000000..365c610aa --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/pom.xml @@ -0,0 +1,110 @@ + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-framework-migrate-server + + + + com.webank.wedatasphere.dss + dss-framework-project-server + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-framework-orchestrator-server + ${dss.version} + provided + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.linkis + linkis-storage + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-orchestrator-common + ${dss.version} + provided + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.xml + **/*.properties + **/*.yml + + + + + + + + \ No newline at end of file diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/conf/MigrateConf.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/conf/MigrateConf.java new file mode 100644 index 000000000..2a46b8f41 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/conf/MigrateConf.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.migrate.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class MigrateConf { + + public static final String ORC_SERVER_NAME = + CommonVars.apply("wds.dss.framework.migrate.orc.name", "dss-framework-orchestrator-server-dev").getValue(); + + public static final String WORKFLOW_SERVER_NAME = + CommonVars.apply("wds.dss.workflow.server.name", "dss-workflow-server-dev").getValue(); +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/exception/MigrateErrorException.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/exception/MigrateErrorException.java new file mode 100644 index 000000000..a3de62e99 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/exception/MigrateErrorException.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.migrate.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class MigrateErrorException extends ErrorException { + + + public MigrateErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public MigrateErrorException(int errCode, String desc, Throwable throwable){ + this(errCode, desc); + this.initCause(throwable); + } + +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/restful/DSSMigrateRestful.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/restful/DSSMigrateRestful.java new file mode 100644 index 000000000..3531c55bf --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/restful/DSSMigrateRestful.java @@ -0,0 +1,691 @@ +package com.webank.wedatasphere.dss.migrate.restful; + +import com.google.common.collect.Lists; +import com.webank.wedatasphere.dss.common.entity.IOType; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelKeyConvertor; +import com.webank.wedatasphere.dss.common.protocol.RequestQueryWorkFlow; +import com.webank.wedatasphere.dss.common.protocol.ResponseExportOrchestrator; +import com.webank.wedatasphere.dss.common.utils.*; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; +import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; +import com.webank.wedatasphere.dss.framework.project.server.service.BMLService; +import com.webank.wedatasphere.dss.framework.project.service.DSSFrameworkProjectService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.migrate.conf.MigrateConf; +import com.webank.wedatasphere.dss.migrate.exception.MigrateErrorException; +import com.webank.wedatasphere.dss.migrate.service.MetaService; +import com.webank.wedatasphere.dss.migrate.service.MigrateService; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.*; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.builder.impl.SSOUrlBuilderOperationImpl; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseQueryWorkflow; +import com.webank.wedatasphere.dss.workflow.core.WorkflowFactory; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.json2flow.JsonToFlowParser; +import org.apache.commons.collections.CollectionUtils; +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.common.exception.LinkisException; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.*; +import java.util.*; + +@RequestMapping(path = "/dss/framework/release", produces = {"application/json"}) +@RestController +public class DSSMigrateRestful { + + private static final Logger LOG = LoggerFactory.getLogger(DSSMigrateRestful.class); + public static final int DEFAULT_PROJECT_AREA = 0; // 运营优化 + public static final String PROJECT_EDIT_USER_KEY = "editUsers"; + public static final String PROJECT_ACCESS_USER_KEY = "accessUsers"; + public static final String PROJECT_RELEASE_USER_KEY = "releaseUsers"; + public static final String EXPORT_DEFAULT_FILE_NAME = "default_orc"; + public static final String FLOW_ID_KEY = "flowId"; + public static final String FLOW_BML_VERSION_KEY = "bmlVersion"; + public static final String DEFAULT_PROJECT_ORCHESTRATOR_MODE = "pom_work_flow"; + + @Autowired + private MigrateService migrateService; + @Autowired + private DSSWorkspaceService dssWorkspaceService; + @Autowired + private DSSProjectService dssProjectService; + @Autowired + private DSSFrameworkProjectService dssFrameworkProjectService; + @Autowired + BMLService bmlService; + @Autowired + private MetaService metaService; + + // todo only dev开发中心 + private Sender orchestratorSender = Sender.getSender(MigrateConf.ORC_SERVER_NAME); + private Sender workflowSender = Sender.getSender(MigrateConf.WORKFLOW_SERVER_NAME); + + + @PostMapping("/importOldDSSProject") + public Message importOldDSSProject(HttpServletRequest req, + @RequestParam(value = "dssLabels", required = false) String dssLabels, + @RequestParam(name = "file") List files) throws Exception { + //通过从0.x导出的zip包上传到此处,然后进行一下导入到开发环境 + //1.下载到本地解压 + //2.保证工程信息要同步,如果没有建工程,那么就要把工程给建立 + //2.丰富相关内容,适配1.0 + //3.将适配之后的目录打包成新的zip包 + //4.上传到bml + //5.通过resourceId 和 version 导入到 dev的 orchestrator-server + String userName = SecurityFilter.getLoginUsername(req); +// List files = form.getFields("file"); + if (files == null || files.size() <= 0) { + LOG.error("files are null, can not continue"); + return Message.error("no files to import"); + } + //只取第一个文件 + MultipartFile p = files.get(0); + String fileName = new String(p.getOriginalFilename().getBytes("ISO8859-1"), "UTF-8"); + InputStream is = null; + OutputStream os = null; + try { + String inputPath = IoUtils.generateIOPath(userName, "input", fileName); + File file = new File(inputPath); + if (file.getParentFile().exists()) { + FileUtils.deleteDirectory(file.getParentFile()); + } + is = p.getInputStream(); + os = IoUtils.generateExportOutputStream(inputPath); + IOUtils.copy(is, os); +// Workspace workspace = new Workspace(); + Cookie[] cookies = req.getCookies(); + Workspace workspace = getWorkspace(req); + migrateService.migrate(userName, inputPath, workspace); + } catch (Exception e) { + String errorMsg = "project import failed for:" + e.getMessage(); + Integer errorCode = 40035; + if (e instanceof DSSProjectErrorException) { + DSSProjectErrorException errorException = (DSSProjectErrorException) e; + errorCode = errorException.getErrCode(); + errorMsg = errorException.getDesc(); + } else if (e instanceof MigrateErrorException) { + MigrateErrorException errorException = (MigrateErrorException) e; + errorCode = errorException.getErrCode(); + errorMsg = errorException.getDesc(); + } else if (e instanceof LinkisException) { + LinkisException errorException = (LinkisException) e; + errorCode = errorException.getErrCode(); + errorMsg = errorException.getDesc(); + } + LOG.error("fatal error:{}", e.getMessage(), e); + throw new MigrateErrorException(errorCode, errorMsg); + } finally { + IOUtils.closeQuietly(os); + IOUtils.closeQuietly(is); + } + return Message.ok(); + } + + public static Workspace getWorkspace(HttpServletRequest request) { + Workspace workspace = new Workspace(); + Cookie[] cookies = request.getCookies(); + Arrays.stream(cookies).forEach(cookie -> { + if ("workspaceId".equals(cookie.getName())) { + workspace.setWorkspaceName(cookie.getValue()); + } else if ("workspaceName".equals(cookie.getName())) { + workspace.setWorkspaceId(Long.parseLong(cookie.getValue())); + } + }); + workspace.setWorkspaceId(100); + workspace.setWorkspaceName("test_workspace"); + SSOHelper.addWorkspaceInfo(request, workspace); + return workspace; + } + + public static Workspace getWorkspaceForOldVersion(HttpServletRequest request) { + Cookie[] cookies = request.getCookies(); + Cookie workspaceCookie = Arrays.stream(cookies). + filter(cookie -> "workspaceId".equals(cookie.getName())).findAny().orElse(null); + Workspace workspace = new Workspace(); + if (workspaceCookie != null) { + workspace.setWorkspaceName(workspaceCookie.getValue()); + } + SSOUrlBuilderOperation ssoUrlBuilderOperation = new SSOUrlBuilderOperationImpl(); + Arrays.stream(cookies).forEach(cookie -> ssoUrlBuilderOperation.addCookie(cookie.getName(), cookie.getValue())); + String gateWayUrl = Configuration.GATEWAY_URL().getValue(); + String forwardedHost = request.getHeader("X-Forwarded-Host"); + if (StringUtils.isNotEmpty(forwardedHost) && forwardedHost.contains(":")) { + gateWayUrl = "http://" + forwardedHost + "/"; + } + ssoUrlBuilderOperation.setDSSUrl(gateWayUrl); + ssoUrlBuilderOperation.setWorkspace(workspace.getWorkspaceName()); +// workspace.setSSOUrlBuilderOperation(ssoUrlBuilderOperation); + return workspace; + } + + + /** + * 导入从dss1.0接口导出的工作流 + * + * @param req + * @param workspaceName 工作空间必须已有,找不到就报错 + * @param projectName 项目名没有就创建 + * @param projectUser 项目用户,接口会保证本用户加入到权限列表。格式跟createProject接口的用户一致 {"editUsers":["alexyang"],"accessUsers":["alexyang"],"releaseUsers":["alexyang"]} + * @param dssLabels + * @return + * @throws Exception + */ + @PostMapping("/importworkflow") + public Message importWorkFlow(HttpServletRequest req, + HttpServletResponse response, + @RequestParam(value = "workspaceName", required = false) String workspaceName, + @RequestParam(value = "projectName", required = false) String projectName, + @RequestParam(value = "projectUser", required = false) String projectUser, + @RequestParam(value = "flowName", required = false) String flowName, + @RequestParam(value = "dssLabels", required = false) String dssLabels, + @RequestParam(name = "file") List files) throws Exception { + String userName = SecurityFilter.getLoginUsername(req); +// List files = form.getFields("file"); + if (files == null || files.size() <= 0) { + LOG.error("files are null, can not continue"); + return Message.error("no files to import"); + } + //只取第一个文件 +// FormDataBodyPart p = files.get(0); + MultipartFile p = files.get(0); +// FormDataContentDisposition fileDetail = p.getFormDataContentDisposition(); +// String fileName = new String(fileDetail.getFileName().getBytes("ISO8859-1"), "UTF-8"); + String fileName = new String(p.getOriginalFilename().getBytes("ISO8859-1"), "UTF-8"); + InputStream is = null; + OutputStream os = null; + Message responseMsg = Message.ok(); + + try { + String pathRoot = IoUtils.generateIOPath(userName, "input", ""); + // todo 注意,此方法要求导出时内部工作流目录名称为 default_orc + String inputZipPath = pathRoot + fileName; + File file = new File(inputZipPath); + if (file.getParentFile().exists()) { + FileUtils.deleteDirectory(file.getParentFile()); + } + is = p.getInputStream(); + os = IoUtils.generateExportOutputStream(inputZipPath); + IOUtils.copy(is, os); + // 获取工作空间id,没有就报错 + List workspaces = dssWorkspaceService.getWorkspaces(userName); + int workspaceId = 0; + DSSWorkspace dssWorkspace = workspaces.stream().filter((w) -> w.getName().equals(workspaceName)).findFirst().get(); + if (null == dssWorkspace) { + LOG.error("Cannot find workspace {}, please create it first.", workspaceName); + throw new MigrateErrorException(0, "Cannot find workspace " + workspaceName); + } + workspaceId = dssWorkspace.getId(); + // todo check +// Workspace workspace = SSOHelper.setWorkspaceId(req, response, workspaceId, workspaceName); + Workspace workspace = SSOHelper.setAndGetWorkspace(req, response, workspaceId, workspaceName); + + // 获取工程,没有就创建 + DSSProjectDO projectDO = getOrCreateProject(projectName, projectUser, userName, workspace, workspaceId); + if (null == projectDO) { + LOG.error("Create project : {} failed.", projectName); + throw new MigrateErrorException(0, "createProject : " + projectName + " failed."); + } + DSSProjectVo projectVo = new DSSProjectVo(); + projectVo.setId(projectDO.getId()); + projectVo.setName(projectDO.getName()); + projectVo.setDescription(projectDO.getDescription()); + + // todo 读取Orchestrator并替换,生成新的zip包 + ZipHelper.unzip(inputZipPath); + String inputPath = pathRoot + EXPORT_DEFAULT_FILE_NAME; + List dssOrchestratorInfos = metaService.readOrchestrator(inputPath); + DSSOrchestratorInfo orchestratorInfo = dssOrchestratorInfos.get(0); + String orcPath = inputPath; +// String orcPath = IoUtils.generateIOPath(userName, "input", "default_orc"); +// mkdir(orcPath); + orchestratorInfo.setProjectId(projectVo.getId()); + orchestratorInfo.setName(flowName); + String oldUUID = migrateService.queryOrcUUIDByName(new Long(dssWorkspace.getId()), projectVo.getId(), flowName); + if (null != oldUUID) { + orchestratorInfo.setUUID(oldUUID); + } else { + String uuid = UUID.randomUUID().toString(); + orchestratorInfo.setUUID(uuid); + } + + //标记当前导出为orchestrator导出 + IoUtils.generateIOType(IOType.ORCHESTRATOR, orcPath); + //标记当前导出环境env + IoUtils.generateIOEnv(orcPath); + metaService.writeOrchestratorInfo(orchestratorInfo, orcPath); + String orcZipPath = ZipHelper.zip(orcPath); + InputStream inputStream = new FileInputStream(orcZipPath); + long importOrcId = 0L; + try { + Map uploadMap = bmlService.upload(userName, inputStream, "default_orc.zip", projectVo.getName()); + String resourceId = uploadMap.get("resourceId").toString(); + String version = uploadMap.get("version").toString(); + //不能走release的importservice接口 因为dev标签没有import操作 + importOrcId = migrateService.importOrcToOrchestrator(resourceId, version, projectVo, userName, "dev", workspace, orchestratorInfo); + } finally { + IOUtils.closeQuietly(inputStream); + } + if (importOrcId <= 0) { + LOG.error("Invalid orcId : {} after imported. User : {}, workspace : {}, projectName : {}, projectId : {}", importOrcId, userName, workspaceName, projectName, projectVo.getId()); + } + + // 获取flowId和bmlVersion以供执行 + DSSOrchestratorVersion orchestratorLatestVersion = getLatestOrchestratorVersion(projectVo.getId(), importOrcId, userName); + if (null != orchestratorLatestVersion) { + responseMsg.data(FLOW_ID_KEY, orchestratorLatestVersion.getAppId()); + String flowLatestBmlVersion = getLatestFlowBmlVersion(userName, orchestratorLatestVersion.getAppId()); + responseMsg.data(FLOW_BML_VERSION_KEY, flowLatestBmlVersion); + } else { + LOG.error("Got null orchestrator version after imported. User : {}, workspace : {}, projectName : {}, projectId : {}", importOrcId, userName, workspaceName, projectName, projectVo.getId()); + responseMsg.data(FLOW_ID_KEY, null); + responseMsg.data(FLOW_BML_VERSION_KEY, null); + } + + } catch (Exception e) { + String errorMsg = "project import failed for:" + e.getMessage(); + Integer errorCode = 40035; + if (e instanceof DSSProjectErrorException) { + DSSProjectErrorException errorException = (DSSProjectErrorException) e; + errorCode = errorException.getErrCode(); + errorMsg = errorException.getDesc(); + } else if (e instanceof MigrateErrorException) { + MigrateErrorException errorException = (MigrateErrorException) e; + errorCode = errorException.getErrCode(); + errorMsg = errorException.getDesc(); + } else if (e instanceof LinkisException) { + LinkisException errorException = (LinkisException) e; + errorCode = errorException.getErrCode(); + errorMsg = errorException.getDesc(); + } + LOG.error("fatal error:{}", e.getMessage(), e); + throw new MigrateErrorException(errorCode, errorMsg); + } finally { + IOUtils.closeQuietly(os); + IOUtils.closeQuietly(is); + } + return responseMsg; + } + + /** + * 下载资源文件 + * 1、下载编排文到本地临时目录下 + * 2、解压文件,解析json文件,获取工作流节点对象 + * 3、替换资源文件的名称为节点名称+版本号 + * 4、删除无用文件 + * 5、将文件打成zip包并输出 + * 6、清理掉文件夹 + * + * @param req reqeust + * @param resp response + * @param outputFileName 下载文件名称 + * @param charset 文件字符集 + * @param outputFileType 输出文件类型 + * @param projectName 工程名称 + * @param orchestratorId 编排ID + * @param orcVersionId 编排版本号ID + * @param addOrcVersion 是否增加版本号 + * @param labels 标签 + * @throws DSSErrorException + * @throws IOException + */ + @RequestMapping(path = "exportOrcSqlFile", method = RequestMethod.GET) + public void exportOrcSqlFile(HttpServletRequest req, + HttpServletResponse resp, + @RequestParam(defaultValue = "exportOrc", required = false, name = "outputFileName") String outputFileName, + @RequestParam(defaultValue = "utf-8", required = false, name = "charset") String charset, + @RequestParam(defaultValue = "zip", required = false, name = "outputFileType") String outputFileType, + @RequestParam(name = "projectName") String projectName, + @RequestParam(name = "orchestratorId") Long orchestratorId, + @RequestParam(required = false, name = "orcVersionId") Long orcVersionId, + @RequestParam(defaultValue = "false", required = false, name = "addOrcVersion") Boolean addOrcVersion, + @RequestParam(required = false, name = "labels") String labels) throws DSSErrorException, IOException { + + resp.addHeader("Content-Disposition", "attachment;filename=" + + new String(outputFileName.getBytes("UTF-8"), "ISO8859-1") + "." + outputFileType); + resp.setCharacterEncoding(charset); + Workspace workspace = SSOHelper.getWorkspace(req); + String userName = SecurityFilter.getLoginUsername(req); + List dssLabelList = getDSSLabelList(labels); + ResponseExportOrchestrator exportResponse = null; + OrchestratorVo orchestratorVo; + if (orcVersionId != null) { + orchestratorVo = (OrchestratorVo) orchestratorSender.ask(new RequestQueryByIdOrchestrator(orchestratorId, orcVersionId)); + } else { + orchestratorVo = (OrchestratorVo) orchestratorSender.ask(new RequestQueryByIdOrchestrator(orchestratorId, null)); + } + orcVersionId = orchestratorVo.getDssOrchestratorVersion().getId(); + LOG.info("export orchestrator orchestratorId " + orchestratorId + ",orcVersionId:" + orcVersionId); + try { + RequestExportOrchestrator requestExportOrchestrator = new RequestExportOrchestrator( + userName, orchestratorId, orcVersionId, projectName, dssLabelList, addOrcVersion, workspace); + exportResponse = (ResponseExportOrchestrator) orchestratorSender.ask(requestExportOrchestrator); + } catch (Exception e) { + LOG.error("export orchestrator failed for ", e); + throw new DSSErrorException(100789, "export orchestrator failed for " + e.getMessage()); + } + if (null != exportResponse) { + // 获取本地路径 + String pathRoot = IoUtils.generateIOPath(userName, "project", ""); + // 下载编排文件到本地 + String zipFilePath = bmlService.downloadToLocalPath(userName, exportResponse.resourceId(), exportResponse.version(), pathRoot + "project.zip"); + // 解压下载文件 + ZipHelper.unzip(zipFilePath); + // 解压后工程文件夹路径 + String projectBasePath = pathRoot + "default_orc"; + // 解压工作流zip文件 + String orcFlowZipFile = projectBasePath + File.separator + "orc_flow.zip"; + ZipHelper.unzip(orcFlowZipFile); + // 解压后工作流文件夹路径 + String flowBasePath = projectBasePath + File.separator + projectName; + // 读取工作流信息 + List dssFlows = metaService.readFlow(flowBasePath); + // 获取工作流节点解析对象 + JsonToFlowParser jsonToFlowParser = WorkflowFactory.INSTANCE.getJsonToFlowParser(); + List uploadResourceNewFileNameList = new ArrayList<>(); + for (DSSFlow dssFlow : dssFlows) { + String workflowPath = flowBasePath + File.separator + dssFlow.getName() + File.separator; + String flowJsonFile = workflowPath + dssFlow.getName() + ".json"; + String jsonContent = FileHelper.readFile(flowJsonFile); + + if (StringUtils.isNotBlank(jsonContent)) { + //将json读取为string,存入workflow + dssFlow.setFlowJson(jsonContent); + // 获取工作流节点信息 + Workflow workflow = jsonToFlowParser.parse(dssFlow); + // 替换用户上传的资源文件名称 + List uploadResources = workflow.getFlowResources(); + if (uploadResources != null && uploadResources.size() > 0) { + for (Resource uploadResource : uploadResources) { + String oldUploadResourceName = workflowPath + "resource" + File.separator + uploadResource.getResourceId() + ".re"; + String newUploadResourceName = workflowPath + "resource" + File.separator + uploadResource.getFileName(); + FileUtils.getFile(oldUploadResourceName).renameTo(new File(newUploadResourceName)); + uploadResourceNewFileNameList.add(newUploadResourceName); + } + } + // 替换脚本sql文件名称为节点名称 + workflow.getWorkflowNodes().forEach(workflowNode -> { + List resources = workflowNode.getDSSNode().getResources(); + if (resources != null && resources.size() > 0) { + for (Resource resource : resources) { + String oldSqlFilePath = workflowPath + "resource" + File.separator + resource.getResourceId() + "_" + resource.getVersion() + ".re"; + String newSqlFilePath = workflowPath + "resource" + File.separator + workflowNode.getDSSNode().getName() + "_" + resource.getVersion() + ".re"; + FileUtils.getFile(oldSqlFilePath).renameTo(new File(newSqlFilePath)); + } + } + }); + } + } + //删除掉无用文件,包括zip包,.json,.txt,.properties + List fileNameList = new ArrayList<>(); + FileHelper.getAllFileNames(pathRoot, fileNameList); + fileNameList.forEach(fileName -> { + if (fileName.endsWith(".zip") || fileName.endsWith(".json") || fileName.endsWith(".txt") || fileName.endsWith(".properties")) { + // 上传的资源文件不能删除 + if (uploadResourceNewFileNameList.stream().noneMatch(fileName::equals)) { + new File(fileName).delete(); + } + } + }); + //将文件打成zip包并输出 + String orcZipPath = ZipHelper.zip(flowBasePath, true); + try (InputStream inputStream = new FileInputStream(orcZipPath)) { + IOUtils.copy(inputStream, resp.getOutputStream()); + resp.getOutputStream().flush(); + } catch (IOException e) { + LOG.error("资源文件打包下载失败,下载路径:{}", orcZipPath, e); + throw new DSSErrorException(100800, "资源文件打包下载失败:原因: " + e.getMessage()); + } + } + } + + + //生成label list + private List getDSSLabelList(String labels) { + String labelStr = DSSCommonUtils.ENV_LABEL_VALUE_DEV; + try { + Map labelMap = DSSCommonUtils.COMMON_GSON.fromJson(labels, Map.class); + if (labelMap.containsKey(LabelKeyConvertor.ROUTE_LABEL_KEY)) { + labelStr = (String) labelMap.get(LabelKeyConvertor.ROUTE_LABEL_KEY); + } + } catch (Exception e) { + LOG.error("get labels failed for {}", e.getMessage()); + } + List dssLabelList = Arrays.asList(new EnvDSSLabel(labelStr)); + return dssLabelList; + } + + /** + * 查找或创建工程 + * + * @param projectName + * @param projectUserJson {"editUsers":["alexyang"],"accessUsers":["alexyang"],"releaseUsers":["alexyang"]} + * @return + * @throws MigrateErrorException + */ + private DSSProjectDO getOrCreateProject(String projectName, String projectUserJson, String username, Workspace workspace, long workspaceId) throws Exception { + if (StringUtils.isBlank(projectName) || StringUtils.isBlank(username)) { + throw new MigrateErrorException(0, "Invalid projectName : " + projectName + " , or username : " + username); + } + DSSProjectDO project = dssProjectService.getProjectByName(projectName); + if (null == project) { + ProjectCreateRequest request = new ProjectCreateRequest(); + request.setName(projectName); + request.setApplicationArea(DEFAULT_PROJECT_AREA); + request.setWorkspaceId(workspaceId); + request.setWorkspaceName(workspace.getWorkspaceName()); + request.setDescription("dss auto test."); + + Map userMap = BDPJettyServerHelper.gson().fromJson(projectUserJson, HashMap.class); + Set accessUsers = new HashSet<>(1); + accessUsers.add(username); + if (userMap.containsKey(PROJECT_ACCESS_USER_KEY)) { + List acceList = (List) userMap.get(PROJECT_ACCESS_USER_KEY); + accessUsers.addAll(acceList); + acceList.clear(); + acceList.addAll(accessUsers); + request.setAccessUsers(acceList); + } + Set editUsers = new HashSet<>(1); + editUsers.add(username); + if (userMap.containsKey(PROJECT_EDIT_USER_KEY)) { + List editList = (List) userMap.get(PROJECT_EDIT_USER_KEY); + editUsers.addAll(editList); + editList.clear(); + editList.addAll(editUsers); + request.setEditUsers(editList); + } + Set releaseUsers = new HashSet<>(1); + releaseUsers.add(username); + if (userMap.containsKey(PROJECT_RELEASE_USER_KEY)) { + List releList = (List) userMap.get(PROJECT_RELEASE_USER_KEY); + releaseUsers.addAll(releList); + releList.clear(); + releList.addAll(releaseUsers); + } + request.setDevProcessList(Lists.newArrayList("dev", "prod")); + request.setOrchestratorModeList(Lists.newArrayList("pom_work_flow")); + DSSProjectVo projectVo = dssFrameworkProjectService.createProject(request, username, workspace); + project = dssProjectService.getProjectByName(projectName); + } + return project; + } + + private DSSOrchestratorVersion getLatestOrchestratorVersion(Long projectId, Long orcId, String userName) { + RequestOrchestratorVersion requestOrchestratorVersion = new RequestOrchestratorVersion(); + requestOrchestratorVersion.setProjectId(projectId); + requestOrchestratorVersion.setOrchestratorId(orcId); + requestOrchestratorVersion.setUsername(userName); + ResponseOrchetratorVersion orchetratorVersion = (ResponseOrchetratorVersion) orchestratorSender.ask(requestOrchestratorVersion); + DSSOrchestratorVersion orchestratorLatestVersion = orchetratorVersion.getOrchestratorVersions().stream() + .filter((v) -> v.getValidFlag() == 1).sorted(new Comparator() { + @Override + public int compare(DSSOrchestratorVersion o1, DSSOrchestratorVersion o2) { + // 注意是逆序 + if (o1.getVersion().compareToIgnoreCase(o2.getVersion()) < 0) { + return 1; + } else if (o1.getVersion().compareToIgnoreCase(o2.getVersion()) == 0) { + return 0; + } else { + return -1; + } + } + }).findFirst().get(); + return orchestratorLatestVersion; + } + + private String getLatestFlowBmlVersion(String username, long flowId) { + RequestQueryWorkFlow requestQueryWorkFlow = new RequestQueryWorkFlow(username, flowId); + ResponseQueryWorkflow responseQueryWorkflow = (ResponseQueryWorkflow) workflowSender.ask(requestQueryWorkFlow); + if (null != responseQueryWorkflow && null != responseQueryWorkflow.getDssFlow()) { + return responseQueryWorkflow.getDssFlow().getBmlVersion(); + } else { + return null; + } + } + + /** + * 批量导出工程下所有工作流到指定本地目录。导出机器在响应需求的project server上 + * + * @param req + * @param response + * @return + * @throws Exception + */ + @RequestMapping(path = "/exportallflow", method = RequestMethod.POST) + public Message exportAllFlowInProject(HttpServletRequest req, + HttpServletResponse response, + @RequestParam(name = "workspaceName") String workspaceName, + @RequestParam(name = "projectName") String projectName, + @RequestParam(name = "pathRoot") String pathRoot) throws Exception { + String username = SecurityFilter.getLoginUsername(req); + + // 1,获取项目下所有的工作流 + // 获取工作空间id,没有就报错 + List workspaces = dssWorkspaceService.getWorkspaces(username); + long workspaceId = 0l; + DSSWorkspace dssWorkspace = workspaces.stream().filter((w) -> w.getName().equals(workspaceName)).findFirst().get(); + if (null == dssWorkspace) { + LOG.error("Cannot find workspace {}, please create it first.", workspaceName); + throw new MigrateErrorException(0, "Cannot find workspace " + workspaceName); + } + workspaceId = dssWorkspace.getId(); + Workspace workspace = SSOHelper.setAndGetWorkspace(req, response, workspaceId, workspaceName); + // 获取项目id,没有就报错 + if (StringUtils.isBlank(projectName) || StringUtils.isBlank(username)) { + throw new MigrateErrorException(0, "Invalid projectName : " + projectName + " , or username : " + username); + } + DSSProjectDO projectDo = dssProjectService.getProjectByName(projectName); + if (null == projectDo) { + throw new MigrateErrorException(0, "cannot find projectName : " + projectName + " in workspace : " + workspaceName); + } + // 获取工作流列表 + RequestOrchestratorInfos requestOrchestratorInfos = new RequestOrchestratorInfos(); + requestOrchestratorInfos.setWorkspaceId(workspaceId); + requestOrchestratorInfos.setProjectId(projectDo.getId()); + // mod只能为一个 + Arrays.stream(projectDo.getOrchestratorMode().split(",")).forEach((mode) -> { + if (StringUtils.isNotBlank(mode)) { + requestOrchestratorInfos.setOrchestratorMode(mode); + } + }); + if (StringUtils.isBlank(requestOrchestratorInfos.getOrchestratorMode())) { + requestOrchestratorInfos.setOrchestratorMode(DEFAULT_PROJECT_ORCHESTRATOR_MODE); + } + ResponseOrchestratorInfos responseOrchestratorInfos = (ResponseOrchestratorInfos) orchestratorSender.ask(requestOrchestratorInfos); + int count = 0; + if (CollectionUtils.isNotEmpty(responseOrchestratorInfos.getOrchestratorInfos())) { + for (DSSOrchestratorInfo orchestratorInfo : responseOrchestratorInfos.getOrchestratorInfos()) { + List devLabels = Lists.newArrayList(new EnvDSSLabel("dev")); + // 查询最新Orchestrator版本 + RequestOrchestratorVersion requestOrchestratorVersion = new RequestOrchestratorVersion(); + requestOrchestratorVersion.setUsername(username); + requestOrchestratorVersion.setOrchestratorId(orchestratorInfo.getId()); + requestOrchestratorVersion.setProjectId(orchestratorInfo.getProjectId()); + ResponseOrchetratorVersion responseOrchetratorVersion = null; + try { + responseOrchetratorVersion = (ResponseOrchetratorVersion) orchestratorSender.ask(requestOrchestratorVersion); + } catch (Exception e) { + DSSExceptionUtils.dealErrorException(60015, "Ask orchestrotor version failed " + BDPJettyServerHelper.gson().toJson(requestOrchestratorVersion), e, + DSSErrorException.class); + } + long latestVersionId = responseOrchetratorVersion.getOrchestratorVersions().stream().map((version) -> version.getId()).max( + new Comparator() { + @Override + public int compare(Long o1, Long o2) { + // 注意,正序 + if (o1 < o2) { + return -1; + } else if (o1 == o2) { + return 0; + } else { + return 1; + } + } + } + ).get(); + exportFlow(username, workspace, projectName, orchestratorInfo.getId(), latestVersionId, + false, devLabels, orchestratorInfo.getName(), pathRoot); + count += 1; + } + } + Message respMsg = Message.ok(); + respMsg.data("count", count); + respMsg.data("location", pathRoot); + respMsg.data("serviceInstance", Sender.getThisInstance()); + return respMsg; + } + + private void exportFlow(String username, Workspace workspace, String projectName, long orchestratorId, + long orchestratorVersionId, boolean addOrcVersion, List labels, + String flowName, String pathRoot) throws Exception { + RequestExportOrchestrator exportRequest = new RequestExportOrchestrator(username, + orchestratorId, orchestratorVersionId, + projectName, labels, addOrcVersion, workspace); + ResponseExportOrchestrator exportResponse = null; + try { + exportResponse = (ResponseExportOrchestrator) orchestratorSender.ask(exportRequest); + } catch (Exception e) { + DSSExceptionUtils.dealErrorException(60015, "export orchestrator ref failed " + BDPJettyServerHelper.gson().toJson(exportRequest), e, + DSSErrorException.class); + } + LOG.info("End to ask to export orchestrator, responseRef is {}", DSSCommonUtils.COMMON_GSON.toJson(exportResponse)); + if (exportResponse == null) { + LOG.error("exportResponse is null, it means export is failed"); + DSSExceptionUtils.dealErrorException(63323, "exportResponse is null, it means export is failed", DSSErrorException.class); + } + String resourceId = exportResponse.resourceId(); + String version = exportResponse.version(); + bmlService.downloadToLocalPath(username, resourceId, version, pathRoot + "/" + projectName + "/" + flowName + ".zip"); + } + + +} \ No newline at end of file diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/MetaService.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/MetaService.java new file mode 100644 index 000000000..2a3e4cd2e --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/MetaService.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.migrate.service; + + +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; + +import java.io.IOException; +import java.util.List; + +public interface MetaService { + + DSSProjectDO readProject(String basePath) throws IOException; + + List readFlow(String basePath) throws IOException; + + List readOrchestrator(String basePath) throws IOException; + + List readFlowRelation(String basePath) throws IOException; + + void writeOrchestratorInfo(DSSOrchestratorInfo orchestratorInfo, String orcPath) throws IOException; + + void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException; + + +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/MigrateService.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/MigrateService.java new file mode 100644 index 000000000..8c93ae785 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/MigrateService.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.migrate.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; + +public interface MigrateService { + + + void migrate(String userName, String inputZipPath, Workspace workspace) throws Exception; + + long importOrcToOrchestrator(String resourceId, String version, DSSProjectVo project, + String username, String label, Workspace workspace, DSSOrchestratorInfo dssOrchestratorInfo); + + DSSOrchestratorInfo buildOrchestratorInfo(DSSFlow dssFlow, DSSProjectVo dssProject, Long workspaceId) throws DSSErrorException; + + String queryOrcUUIDByName(Long workspaceId,Long projectId,String orcName) throws DSSErrorException; +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaReader.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaReader.java new file mode 100644 index 000000000..aee2d7563 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaReader.java @@ -0,0 +1,165 @@ +package com.webank.wedatasphere.dss.migrate.service.impl; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Field; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +public class MetaReader { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final Class tClass; + private final String tableName; + private String commentPrefix = "#"; + private String seperator = "\\|"; + private List datas = new ArrayList<>(); + private List fields = null; + private List comments = new ArrayList<>(); + private List> body = new ArrayList<>(); + private boolean finded = false; + private boolean firstLine = true; + + public MetaReader(Class tClass, String tableName) { + this.tClass = tClass; + this.tableName = tableName; + } + + public static MetaReader of(String tableName, Class tClass) { + return new MetaReader<>(tClass, tableName); + } + + public List read(InputStream inputStream) throws IOException { + try (InputStreamReader streamReader = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(streamReader);) { + readTable(reader); + } + readT(); + return datas; + } + + public String read(InputStream inputStream, String key) throws IOException { + try (InputStreamReader streamReader = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(streamReader);) { + readTable(reader); + } + String comment = comments.stream().filter(c -> c.contains(key + ":")).findFirst().orElse(""); + String[] split = comment.split(":"); + if (split.length > 1) return split[1]; + return ""; + } + + private void readT() { + body.stream().map(DSSExceptionUtils.map(this::lineToT)).forEach(datas::add); + } + + private T lineToT(List list) throws IllegalAccessException, InstantiationException, NoSuchFieldException, ParseException { + T t = tClass.newInstance(); + for (int i = 0; i < list.size(); i++) { + try{ + String valueStr = list.get(i); + if ("null".equalsIgnoreCase(valueStr)) continue; + Field declaredField = tClass.getDeclaredField(fields.get(i)); + declaredField.setAccessible(true); + Object value = null; + String type = declaredField.getType().getSimpleName(); + switch (type) { + case "String": + value = valueStr; + break; + case "Date": + value = new SimpleDateFormat("EEE MMM dd HH:mm:ss ZZZ yyyy", Locale.ENGLISH).parse(valueStr); + break; + case "Long": + value = Long.valueOf(valueStr); + break; + case "Boolean": + value = Boolean.valueOf(valueStr); + break; + case "Integer": + //由于两个版本字段类型发生变化,先做下适配处理 + if (declaredField.getName().equalsIgnoreCase("isPersonal")) { + if ("true".equals(valueStr)) { + value = Integer.valueOf(1); + } else { + value = Integer.valueOf(0); + } + } else { + value = Integer.valueOf(valueStr); + } + break; + default: + logger.warn(String.format("unsupport type %s", type)); + } + + declaredField.set(t, value); + }catch (Exception e){ + logger.warn("设置对象属性失败", e); + } + + } + return t; + } + + private void readTable(BufferedReader reader) throws IOException { + String line = null; + while ((line = reader.readLine()) != null) { + if (!finded && !shut(line)) { + continue; + } + if (finded && isTableName(line)) { + break; + } + if (shut(line)) { + finded = true; + continue; + } + // TODO: 2020/3/9 + if (isComment(line)) { + comments.add(line); + continue; + } + if (firstLine) { + //handle head + fields = Arrays.stream(line.split(seperator)).collect(Collectors.toList()); + firstLine = false; + continue; + } + body.add(Arrays.stream(line.split(seperator)).collect(Collectors.toList())); + //handle body + } + } + + private boolean isComment(String str) { + return str.startsWith(commentPrefix); + } + + private boolean isTableName(String str) { + return isComment(str) && str.contains(commentPrefix + "tableName:"); + } + + private boolean isClass(String str) { + return isComment(str) && str.contains(commentPrefix + "class:"); + } + + private String getComment(String str) { + return str.substring(1); + } + + private boolean shut(String str) { + return isTableName(str) && getComment(str).equals(String.format("tableName:%s", tableName)); + } +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaServiceImpl.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaServiceImpl.java new file mode 100644 index 000000000..522d7d3a6 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaServiceImpl.java @@ -0,0 +1,110 @@ +package com.webank.wedatasphere.dss.migrate.service.impl; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.migrate.service.MetaService; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; + +@Service +public class MetaServiceImpl implements MetaService { + + private static final String FILE_NAME = "meta.txt"; + + @Override + public DSSProjectDO readProject(String basePath) throws IOException { + try (InputStream inputStream = generateInputstream(basePath)) { + List dwsProjectList = MetaReader.of("dss_project", DSSProjectDO.class).read(inputStream); + return dwsProjectList.get(0); + } + + } + + @Override + public List readFlow(String basePath) throws IOException { + try (InputStream inputStream = generateInputstream(basePath)) { + return MetaReader.of("dss_flow", DSSFlow.class).read(inputStream); + } + } + + @Override + public List readOrchestrator(String basePath) throws IOException { + try (InputStream inputStream = generateInputstream(basePath)) { + return MetaReader.of("dss_orchestrator", DSSOrchestratorInfo.class).read(inputStream); + } + } + + @Override + public List readFlowRelation(String basePath) throws IOException { + try (InputStream inputStream = generateInputstream(basePath)) { + return MetaReader.of("dss_flow_relation", DSSFlowRelation.class).read(inputStream); + } + } + + + @Override + public void writeOrchestratorInfo(DSSOrchestratorInfo orchestratorInfo, String orcPath) throws IOException { + export(orchestratorInfo,orcPath); + + } + + + public void export(DSSOrchestratorInfo dssOrchestratorInfo, String savePath) throws IOException { + + try ( + OutputStream outputStream = generateOutputStream(savePath) + ) { + exportOrchestratorBaseInfo(dssOrchestratorInfo, outputStream); + } + } + + private void exportOrchestratorBaseInfo(DSSOrchestratorInfo dssOrchestratorInfo,OutputStream outputStream) throws IOException { + + MetaWriter.of("dss_orchestrator", DSSOrchestratorInfo.class).data(dssOrchestratorInfo).write(outputStream); + + } + + + private InputStream generateInputstream(String basePath) throws IOException { + return IoUtils.generateInputInputStream(basePath + File.separator + FILE_NAME); + } + + private final String fileName = "meta.txt"; + + + @Override + public void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException { + + try ( + OutputStream outputStream = generateOutputStream(savePath) + ) { + exportFlowBaseInfo(allDSSFlows, outputStream); + exportFlowRelation(allFlowRelations, outputStream); + } + } + + private OutputStream generateOutputStream(String basePath) throws IOException { + return IoUtils.generateExportOutputStream(basePath + File.separator + fileName); + } + + private void exportFlowBaseInfo(List DSSFlows, OutputStream outputStream) throws IOException { + + MetaWriter.of("dss_flow", DSSFlow.class).data(DSSFlows).write(outputStream); + + } + + private void exportFlowRelation(List flowRelations, OutputStream outputStream) throws IOException { + + MetaWriter.of("dss_flow_relation", DSSFlowRelation.class).data(flowRelations).write(outputStream); + + } +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaWriter.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaWriter.java new file mode 100644 index 000000000..fedd4901b --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MetaWriter.java @@ -0,0 +1,159 @@ +package com.webank.wedatasphere.dss.migrate.service.impl; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class MetaWriter { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final Class tClass; + private final String tableName; + private List datas = new ArrayList<>(); + private List ignoreFields = new ArrayList<>(); + private List fields = null; + private List comments = new ArrayList<>(); + private List table = new ArrayList<>(); + private String commentPrefix = "#"; + private String seperator = "|"; + + private CommonVars DSS_EXPORT_ENV = CommonVars.apply("wds.dss.server.export.env", "生产中心"); + + public MetaWriter(Class tClass, String tableName) { + this.tClass = tClass; + this.tableName = tableName; + } + + public static MetaWriter of(String tableName, Class tClass) { + return new MetaWriter<>(tClass, tableName); + } + + public MetaWriter ignore(String... fields) { + ignoreFields.addAll(Arrays.stream(fields).collect(Collectors.toList())); + return this; + } + + public MetaWriter data(List datas) { + this.datas.addAll(datas); + return this; + } + + public MetaWriter data(T data) { + this.datas.add(data); + return this; + } + + public MetaWriter comment(String... comments) { + this.comments.addAll(Arrays.stream(comments).collect(Collectors.toList())); + return this; + } + + /** + * 上传bml + * + * @return + */ + public InputStream write() { + writeComment(); + writeHead(); + writeBody(); + //table 添加换行符进行转流 + String tableStr = table.stream().reduce((a, b) -> a + "\n" + b).orElse("") + "\n"; + logger.info("\n" + tableStr); + return new ByteArrayInputStream(tableStr.getBytes()); + } + + /** + * 写入本地 + * + * @param outputStream + */ + public void write(OutputStream outputStream) throws IOException { + writeComment(); + writeHead(); + writeBody(); + IOUtils.writeLines(table, "\n", outputStream); + } + + private void writeBody() { + datas.forEach(DSSExceptionUtils.handling(this::writeBody)); + } + + private void writeBody(T t) throws NoSuchFieldException, IllegalAccessException { + ArrayList line = new ArrayList<>(); + for (String field : fields) { + Field declaredField = tClass.getDeclaredField(field); + declaredField.setAccessible(true); + Object o = declaredField.get(t); + if (o != null) { + line.add(o.toString()); + } else { + line.add(null); + } + } + table.add(reduce(line)); + } + + private String reduce(List strs) { + return strs.stream().reduce((a, b) -> a + seperator + b).orElse(""); + } + + /** + * 写表的头部 + * 1.过滤ignore的属性 + * 2.驼峰转mysql的 _ (暂时略过) + * 3.写入List table + */ + private void writeHead() { + fields = Arrays.stream(tClass.getDeclaredFields()) + .map(Field::getName) + .filter(n -> !ignoreFields.contains(n)) + .collect(Collectors.toList()); + table.add(reduce(fields)); + } + + /** + * 驼峰转_ + * + * @param str + * @return + */ + private String unCamel(String str) { + // TODO: 2020/3/9 + return null; + } + + /** + * 写comment,包括表名,class名,和外部自定义的comment + */ + private void writeComment() { + table.add(connectCommentPrefix(String.format("tableName:%s", tableName))); + table.add(connectCommentPrefix(String.format("class:%s", tClass.getName()))); + table.add(connectCommentPrefix(String.format("env:%s", DSS_EXPORT_ENV.getValue()))); + comments.stream().map(this::connectCommentPrefix).forEach(table::add); + } + + /** + * str前加上comment标识 + * + * @param str + * @return + */ + private String connectCommentPrefix(String str) { + return commentPrefix + str; + } + +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MigrateServiceImpl.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MigrateServiceImpl.java new file mode 100644 index 000000000..74510be91 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/service/impl/MigrateServiceImpl.java @@ -0,0 +1,324 @@ +package com.webank.wedatasphere.dss.migrate.service.impl; + +import com.google.common.collect.Lists; +import com.webank.wedatasphere.dss.common.entity.IOType; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.protocol.ResponseImportOrchestrator; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; +import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; +import com.webank.wedatasphere.dss.framework.project.server.service.BMLService; +import com.webank.wedatasphere.dss.framework.project.service.DSSFrameworkProjectService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; +import com.webank.wedatasphere.dss.migrate.conf.MigrateConf; +import com.webank.wedatasphere.dss.migrate.exception.MigrateErrorException; +import com.webank.wedatasphere.dss.migrate.service.MetaService; +import com.webank.wedatasphere.dss.migrate.service.MigrateService; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.*; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import org.apache.commons.collections.CollectionUtils; +import org.apache.linkis.rpc.Sender; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.*; +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class MigrateServiceImpl implements MigrateService { + + + private static final Logger LOG = LoggerFactory.getLogger(MigrateServiceImpl.class); + + private static final String WORK_FLOW = "workflow"; + + @Autowired + private MetaService metaService; + + @Autowired + private DSSProjectService dssProjectService; + + @Autowired + DSSFrameworkProjectService dssFrameworkProjectService; + + @Autowired + BMLService bmlService; + + private Sender orchestratorSender = Sender.getSender(MigrateConf.ORC_SERVER_NAME); + + @Override + public void migrate(String userName, String inputZipPath, Workspace workspace) throws Exception { + //解压之后需要替换一下文件内容,可以进行反序列化 + String inputPath = ZipHelper.unzip(inputZipPath); + String metaFilePath = inputPath.endsWith("/") ? inputPath + "meta.txt" : inputPath + "/meta.txt"; + try { + replaceFileStr(metaFilePath, "com.webank.wedatasphere.dss.common.entity.project.DWSProject", + "com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO"); + replaceFileStr(metaFilePath, "com.webank.wedatasphere.dss.common.entity.flow.DWSFlow", + "com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow"); + replaceFileStr(metaFilePath, "com.webank.wedatasphere.dss.server.entity.DWSFlowRelation", + "com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation"); + } catch (final IOException e) { + throw new MigrateErrorException(40012, "failed to replace meta.txt", e); + } + DSSProjectDO dssProject = metaService.readProject(inputPath); + DSSProjectVo finalProject; + DSSProjectDO dbProject = dssProjectService.getProjectByName(dssProject.getName()); + if(!dssProject.getUsername().equalsIgnoreCase(userName)){ + LOG.error("fatal error, project owner is {} ,but export user is {}", dssProject.getUsername(), userName); + throw new MigrateErrorException(40013, "project has been exported by others,not owner "+dssProject.getUsername()); + } + if (dbProject == null) { + //判断如果没有该工程,则开始调用接口来进行工程创建 + LOG.info("dssProject {} is not exist will create it first", dssProject.getName()); + ProjectCreateRequest projectCreateRequest = new ProjectCreateRequest(); + projectCreateRequest.setName(dssProject.getName()); + projectCreateRequest.setDescription(dssProject.getDescription()); + projectCreateRequest.setBusiness(dssProject.getBusiness()); + projectCreateRequest.setEditUsers(StringUtils.isNotBlank(dssProject.getUsername()) ? Lists.newArrayList(dssProject.getUsername()) : Lists.newArrayList(userName)); + projectCreateRequest.setAccessUsers(StringUtils.isNotBlank(dssProject.getUsername()) ? Lists.newArrayList(dssProject.getUsername()) : Lists.newArrayList(userName)); + projectCreateRequest.setReleaseUsers(StringUtils.isNotBlank(dssProject.getUsername()) ? Lists.newArrayList(dssProject.getUsername()) : Lists.newArrayList(userName)); + projectCreateRequest.setWorkspaceId(dssProject.getWorkspaceId()); + projectCreateRequest.setApplicationArea(dssProject.getApplicationArea()); + projectCreateRequest.setDevProcessList(Lists.newArrayList("dev", "prod")); + projectCreateRequest.setOrchestratorModeList(Lists.newArrayList("pom_work_flow")); + //todo + workspace.setWorkspaceId(dssProject.getWorkspaceId()); + workspace.setWorkspaceName(""); + workspace.addCookie("workspaceId", String.valueOf(workspace.getWorkspaceId())); + workspace.addCookie("workspaceName", String.valueOf(workspace.getWorkspaceName())); + finalProject = dssFrameworkProjectService.createProject(projectCreateRequest, userName, workspace); + } else if (!dbProject.getUsername().equals(dssProject.getUsername())) { + LOG.error("fatal error, project {} 已经创建,但是创建人不是 {}", dssProject.getName(), dssProject.getCreateBy()); + throw new MigrateErrorException(40035, "project has been created by others"); + } else if (dbProject.getVisible() != null && dbProject.getVisible().intValue() == 0) { + LOG.error("fatal error, project {} 已经被删除 {}", dssProject.getName()); + throw new MigrateErrorException(40036, "The project has been deleted.Please restore project before exporting"); + } else { + DSSProjectVo dssProjectVo = new DSSProjectVo(); + dssProjectVo.setId(dbProject.getId()); + dssProjectVo.setName(dbProject.getName()); + dssProjectVo.setDescription(dbProject.getDescription()); + finalProject = dssProjectVo; + dssProjectService.modifyOldProject(dssProject, dbProject); + } + List dssFlows = metaService.readFlow(inputPath); + dssFlows.stream().forEach(dssFlow -> dssFlow.setProjectID(finalProject.getId())); + + List dssFlowRelations = metaService.readFlowRelation(inputPath); + List rootFlows = dssFlows.stream().filter(DSSFlow::getRootFlow).collect(Collectors.toList()); + + //根据所有的rootflow创建新的orc信息并且写入到新的meta.txt,但是需要另外搞一个orc的路径 + for (DSSFlow rootFlow : rootFlows) { + + String orcPath = IoUtils.generateIOPath(userName, "input", "default_orc"); + try { + mkdir(orcPath); + //生成flow内容zip + String flowZipPath = orcPath + File.separator + finalProject.getName(); + mkdir(flowZipPath); + + + //由于0.x版本按照工程导出的zip包,所有元数据信息是在一块的,这里需要按照orc一个一个进行筛选出来。 + List rootFlowAndSubFlows = new ArrayList<>(); + genRootFlowAndSubFlows(rootFlow, dssFlows, dssFlowRelations, rootFlowAndSubFlows); + + //flowRelations + List rootFlowAndSubFlowsRelations = new ArrayList<>(); + genRootFlowAndSubFlowsRelations(rootFlow.getId(), dssFlowRelations, rootFlowAndSubFlowsRelations); + + //拷贝flow文件夹 + rootFlowAndSubFlows.stream().forEach(dssFlow -> { + try { + FileUtils.copyDirectoryToDirectory(new File(inputPath + File.separator + dssFlow.getName()), new File(flowZipPath)); + } catch (IOException e) { + LOG.error("拷贝flow失败:" + dssFlow.getName(), e); + } + }); + + + //标记当前导出为flow导出 + IoUtils.generateIOType(IOType.FLOW, flowZipPath); + //标记当前导出环境 + IoUtils.generateIOEnv(flowZipPath); + metaService.exportFlowBaseInfo(rootFlowAndSubFlows, rootFlowAndSubFlowsRelations, flowZipPath); + String orcZipPath = ZipHelper.zip(flowZipPath); + FileUtils.getFile(orcZipPath).renameTo(new File(orcPath + File.separator + "orc_flow.zip")); +// + } catch (Exception e) { + LOG.error("拷贝flow zip 失败", e); + } + DSSOrchestratorInfo orchestratorInfo = buildOrchestratorInfo(rootFlow, finalProject, dssProject.getWorkspaceId()); + //标记当前导出为project导出 + IoUtils.generateIOType(IOType.ORCHESTRATOR, orcPath); + //标记当前导出环境env + IoUtils.generateIOEnv(orcPath); + metaService.writeOrchestratorInfo(orchestratorInfo, orcPath); + String orcZipPath = ZipHelper.zip(orcPath); + InputStream inputStream = new FileInputStream(orcZipPath); + try { + Map uploadMap = bmlService.upload(userName, inputStream, "default_orc.zip", finalProject.getName()); + String resourceId = uploadMap.get("resourceId").toString(); + String version = uploadMap.get("version").toString(); + //不能走release的importservice接口 因为dev标签没有import操作 + importOrcToOrchestrator(resourceId, version, finalProject, userName, "dev", workspace, orchestratorInfo); + } finally { + IOUtils.closeQuietly(inputStream); + } + } + } + + + public static void genRootFlowAndSubFlows(DSSFlow parentFlow, + List sourceFlowList, + List sourceFlowRelations, + List resultFlows) { + resultFlows.add(parentFlow); + List subFlowRelations = sourceFlowRelations.stream().filter(dssFlowRelation -> dssFlowRelation.getParentFlowID().equals(parentFlow.getId())).collect(Collectors.toList()); + List subFlowIds = subFlowRelations.stream().map(dssFlowRelation -> dssFlowRelation.getFlowID()).collect(Collectors.toList()); + if (subFlowIds.size() > 0) { + for (Long subFlowId : subFlowIds) { + + DSSFlow subDssFlow = sourceFlowList.stream().filter(dssFlow -> dssFlow.getId().equals(subFlowId)).findAny().orElse(null); + if (null != subDssFlow) { + resultFlows.add(subDssFlow); + parentFlow.addChildren(subDssFlow); + genRootFlowAndSubFlows(subDssFlow, sourceFlowList, sourceFlowRelations, resultFlows); + } + } + } + } + + + public static void genRootFlowAndSubFlowsRelations(Long parentFlowID, + List sourceFlowRelations, + List resultFlowRelations) { + + List subFlowRelations = sourceFlowRelations.stream().filter(dssFlowRelation -> dssFlowRelation.getParentFlowID().equals(parentFlowID)).collect(Collectors.toList()); + + if (subFlowRelations.size() > 0) { + for (DSSFlowRelation subFlowRelation : subFlowRelations) { + resultFlowRelations.add(subFlowRelation); + genRootFlowAndSubFlowsRelations(subFlowRelation.getFlowID(), sourceFlowRelations, resultFlowRelations); + } + } + } + + @Override + public long importOrcToOrchestrator(String resourceId, String version, DSSProjectVo project, + String username, String label, Workspace workspace, DSSOrchestratorInfo dssOrchestratorInfo) { + LOG.info("begin to import dss0.x workflow {} to orc-dev", dssOrchestratorInfo.getName()); + List dssLabels = Lists.newArrayList(new EnvDSSLabel(label)); + String workspaceStr = DSSCommonUtils.COMMON_GSON.toJson(workspace); + RequestImportOrchestrator requestImportOrchestrator = + new RequestImportOrchestrator(username, project.getName(), + project.getId(), resourceId, version, dssOrchestratorInfo.getName(), dssLabels, workspace); + ResponseImportOrchestrator responseImportOrchestrator = (ResponseImportOrchestrator) this.orchestratorSender.ask(requestImportOrchestrator); + return responseImportOrchestrator.orcId(); + } + + @Override + public DSSOrchestratorInfo buildOrchestratorInfo(DSSFlow dssFlow, DSSProjectVo dssProject, Long workspaceId) throws DSSErrorException { + DSSOrchestratorInfo dssOrchestratorInfo = new DSSOrchestratorInfo(); + dssOrchestratorInfo.setName(dssFlow.getName()); + dssOrchestratorInfo.setComment(dssFlow.getDescription()); + dssOrchestratorInfo.setUses(dssFlow.getUses()); + dssOrchestratorInfo.setProjectId(dssProject.getId()); + dssOrchestratorInfo.setAppConnName(WORK_FLOW); + dssOrchestratorInfo.setType(WORK_FLOW); + dssOrchestratorInfo.setSecondaryType("[pom_work_flow_DAG]"); + dssOrchestratorInfo.setDesc(dssFlow.getDescription()); + String oldUUID = queryOrcUUIDByName(workspaceId, dssProject.getId(), dssFlow.getName()); + if (null != oldUUID) { + dssOrchestratorInfo.setUUID(oldUUID); + } else { + String uuid = UUID.randomUUID().toString(); + dssOrchestratorInfo.setUUID(uuid); + } + dssOrchestratorInfo.setCreator(dssFlow.getCreator()); + + return dssOrchestratorInfo; + } + + @Override + public String queryOrcUUIDByName(Long workspaceId, Long projectId, String orcName) throws DSSErrorException { + String uuid = null; + ResponseOrchestratorInfos responseOrchestratorInfos = (ResponseOrchestratorInfos) orchestratorSender + .ask(new RequestOrchestratorInfos(null, projectId, orcName, null)); + if (CollectionUtils.isNotEmpty(responseOrchestratorInfos.getOrchestratorInfos())) { + uuid = responseOrchestratorInfos.getOrchestratorInfos().get(0).getUUID(); + } + return uuid; + } + + @Deprecated + private List queryExitsOrc(List orcIds) throws DSSErrorException { + RequestQueryOrchestrator queryRequest = new RequestQueryOrchestrator(orcIds); + ResponseQueryOrchestrator queryResponse = null; + try { + queryResponse = (ResponseQueryOrchestrator) this.orchestratorSender.ask(queryRequest); + } catch (Exception e) { + DSSExceptionUtils.dealErrorException(60015, "create orchestrator ref failed", + DSSErrorException.class); + } + if (queryResponse == null) { + LOG.error("query response is null, it is a fatal error"); + return null; + } + LOG.info("End to ask to query orchestrator, responseRef is {}", queryResponse); + return queryResponse.getOrchestratorVoes(); + } + + + private void replaceFileStr(String filepath, String sourceStr, String targetStr) throws IOException { + FileReader fis = null; + FileWriter fout = null; + try { + fis = new FileReader(filepath); + char[] data = new char[1024]; + int rn = 0; + StringBuilder sb = new StringBuilder(); + while ((rn = fis.read(data)) > 0) { + String str = String.valueOf(data, 0, rn); + sb.append(str); + } + String str = sb.toString().replace(sourceStr, targetStr); + fout = new FileWriter(filepath); + fout.write(str.toCharArray()); + } finally { + IOUtils.closeQuietly(fis); + IOUtils.closeQuietly(fout); + } + } + + + public static void mkdir(String path) { + File fd = null; + try { + fd = new File(path); + if (!fd.exists()) { + fd.mkdirs(); + } + } catch (Exception e) { + LOG.error("创建目录失败:" + path, e); + } finally { + fd = null; + } + } +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/pom.xml b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/pom.xml new file mode 100644 index 000000000..d04fad0f8 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/pom.xml @@ -0,0 +1,119 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-framework-orchestrator-publish + + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + compile + + + linkis-common + org.apache.linkis + + + + + + com.webank.wedatasphere.dss + dss-orchestrator-db + ${dss.version} + compile + + + + com.webank.wedatasphere.dss + dss-orchestrator-loader + ${dss.version} + compile + + + org.springframework + spring-tx + ${spring.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-sender-service + ${dss.version} + provided + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + org.springframework + spring-context + ${spring.version} + provided + + + org.apache.commons + commons-math3 + provided + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + org.apache.maven.plugins + maven-jar-plugin + + + + + + \ No newline at end of file diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ConversionDSSOrchestratorPlugin.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ConversionDSSOrchestratorPlugin.java new file mode 100644 index 000000000..390b28e90 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ConversionDSSOrchestratorPlugin.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; +import java.util.Map; + + +public interface ConversionDSSOrchestratorPlugin extends DSSOrchestratorPlugin { + + ResponseOperateOrchestrator convert(String userName, + DSSProject project, + Workspace workspace, + Map orchestrationIdMap, + List dssLabels); +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ExportDSSOrchestratorPlugin.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ExportDSSOrchestratorPlugin.java new file mode 100644 index 000000000..d1d1efb34 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ExportDSSOrchestratorPlugin.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import java.util.List; +import java.util.Map; + + +public interface ExportDSSOrchestratorPlugin extends DSSOrchestratorPlugin { + + /** + * 导出Orchestrator基本信息和工作流基本信息 + */ + Map exportOrchestrator(String userName, + Long orchestratorId, + Long orcVersionId, + String projectName, + List dssLabels, + boolean addOrcVersion, Workspace workspace) throws DSSErrorException; + + Long orchestratorVersionIncrease(Long orcId, + String userName, + String comment, + Workspace workspace, + DSSOrchestratorInfo dssOrchestratorInfo, + String projectName, + List dssLabels) throws DSSErrorException; + + Long addVersionAfterPublish(String userName, Workspace workspace, + Long orchestratorId, Long orcVersionId, String projectName, + List dssLabels,String comment) throws DSSErrorException; + +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ImportDSSOrchestratorPlugin.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ImportDSSOrchestratorPlugin.java new file mode 100644 index 000000000..8332d0d2e --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ImportDSSOrchestratorPlugin.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestImportOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import java.io.IOException; +import java.util.List; + + +public interface ImportDSSOrchestratorPlugin extends DSSOrchestratorPlugin { + + /** + * 导入Orchestrator + * @param requestImportOrchestrator + * @return + * @throws DSSErrorException + * @throws IOException + * @throws ExternalOperationFailedException + */ + Long importOrchestrator(RequestImportOrchestrator requestImportOrchestrator) throws Exception; + + +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/conf/DSSOrchestratorConf.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/conf/DSSOrchestratorConf.java new file mode 100644 index 000000000..5740b6e9b --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/conf/DSSOrchestratorConf.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.conf; + +import org.apache.linkis.common.conf.CommonVars; + + +public class DSSOrchestratorConf { + public static final CommonVars DSS_EXPORT_ENV = CommonVars.apply("wds.dss.server.export.env", "dev"); +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ConversionDSSOrchestratorPluginImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ConversionDSSOrchestratorPluginImpl.java new file mode 100644 index 000000000..d6a36ac5c --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ConversionDSSOrchestratorPluginImpl.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.impl; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestConvertOrchestrations; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.AbstractDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.publish.ConversionDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.linkis.rpc.Sender; + +import java.util.List; +import java.util.Map; + + +public class ConversionDSSOrchestratorPluginImpl extends AbstractDSSOrchestratorPlugin implements ConversionDSSOrchestratorPlugin { + + @Override + public ResponseOperateOrchestrator convert(String userName, + DSSProject project, + Workspace workspace, + Map orchestrationIdMap, + List dssLabels) { + //1、发布DSS编排,如DSS工作流 + RequestConvertOrchestrations requestConvertOrchestrator = new RequestConvertOrchestrations(); + requestConvertOrchestrator.setOrchestrationIdMap(orchestrationIdMap); + requestConvertOrchestrator.setProject(project); + requestConvertOrchestrator.setWorkspace(workspace); + requestConvertOrchestrator.setDSSLabels(dssLabels); + requestConvertOrchestrator.setUserName(userName); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(dssLabels); + return (ResponseOperateOrchestrator) sender.ask(requestConvertOrchestrator); + } +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ExportDSSOrchestratorPluginImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ExportDSSOrchestratorPluginImpl.java new file mode 100644 index 000000000..1977b1027 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ExportDSSOrchestratorPluginImpl.java @@ -0,0 +1,223 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.impl; + +import com.webank.wedatasphere.dss.common.entity.IOType; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.exception.DSSOrchestratorErrorException; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.AbstractDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.core.service.BMLService; +import com.webank.wedatasphere.dss.orchestrator.core.utils.OrchestratorUtils; +import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; +import com.webank.wedatasphere.dss.orchestrator.loader.OrchestratorManager; +import com.webank.wedatasphere.dss.orchestrator.publish.ExportDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.publish.io.export.MetaExportService; +import com.webank.wedatasphere.dss.orchestrator.publish.utils.OrchestrationDevelopmentOperationUtils; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCopyOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.*; +import com.webank.wedatasphere.dss.standard.app.development.service.RefCRUDService; +import com.webank.wedatasphere.dss.standard.app.development.service.RefExportService; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static com.webank.wedatasphere.dss.common.utils.ZipHelper.zipExportProject; + + +@Component +public class ExportDSSOrchestratorPluginImpl extends AbstractDSSOrchestratorPlugin implements ExportDSSOrchestratorPlugin { + + static final String DEFAULT_ORC_NAME = "default_orc"; + + @Autowired + private BMLService bmlService; + @Autowired + private OrchestratorMapper orchestratorMapper; + @Autowired + private MetaExportService metaExportService; + @Autowired + private ContextService contextService; + @Autowired + private OrchestratorManager orchestratorManager; + + @Override + @Transactional(rollbackFor = Exception.class) + public Map exportOrchestrator(String userName, Long orchestratorId, Long orcVersionId, String projectName, + List dssLabels, boolean addOrcVersion, Workspace workspace) throws DSSErrorException { + //1、导出info信息 + if (orcVersionId == null || orcVersionId < 0){ + LOGGER.info("orchestratorVersionId is {}.", orcVersionId); + //最简单的就是通过orcId来找到最新的versionId + orcVersionId = orchestratorMapper.findLatestOrcVersionId(orchestratorId); + } + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getOrchestratorVersion(orcVersionId); + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(dssOrchestratorVersion.getOrchestratorId()); + if (dssOrchestratorInfo != null) { + String orcExportSaveBasePath = IoUtils.generateIOPath(userName, DEFAULT_ORC_NAME, ""); + try { + Files.createDirectories(Paths.get(orcExportSaveBasePath).toAbsolutePath().normalize()); + //标记当前导出为project导出 + IoUtils.generateIOType(IOType.ORCHESTRATOR, orcExportSaveBasePath); + //标记当前导出环境env + IoUtils.generateIOEnv(orcExportSaveBasePath); + metaExportService.export(dssOrchestratorInfo, orcExportSaveBasePath); + } catch (IOException e) { + LOGGER.error("Failed to export metaInfo in orchestrator server for orc({}) in version {}.", orchestratorId, orcVersionId, e); + DSSExceptionUtils.dealErrorException(60099, "Failed to export metaInfo in orchestrator server.", e, DSSOrchestratorErrorException.class); + } + LOGGER.info("{} 开始导出Orchestrator: {} 版本ID为: {}.", userName, dssOrchestratorInfo.getName(), orcVersionId); + + //2、导出第三方应用信息,如工作流、Visualis、Qualities + DSSOrchestrator dssOrchestrator = orchestratorManager.getOrCreateOrchestrator(userName, workspace.getWorkspaceName(), dssOrchestratorInfo.getType(), + dssLabels); + ExportResponseRef responseRef = OrchestrationDevelopmentOperationUtils.tryOrchestrationOperation(dssOrchestratorInfo, dssOrchestrator, userName, + workspace, dssLabels, DevelopmentIntegrationStandard::getRefExportService, + developmentService -> ((RefExportService) developmentService).getRefExportOperation(), + null, + projectRefRequestRef -> projectRefRequestRef.setProjectName(projectName).setRefProjectId(dssOrchestratorVersion.getProjectId()), + (developmentOperation, developmentRequestRef) -> { + RefJobContentRequestRef requestRef = (RefJobContentRequestRef) developmentRequestRef; + requestRef.setRefJobContent(MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, dssOrchestratorVersion.getAppId())); + return ((RefExportOperation) developmentOperation).exportRef(requestRef); + }, "export"); + String resourceId = (String) responseRef.getResourceMap().get(ImportRequestRef.RESOURCE_ID_KEY); + String version = (String) responseRef.getResourceMap().get(ImportRequestRef.RESOURCE_VERSION_KEY); + bmlService.downloadToLocalPath(userName, resourceId, version, orcExportSaveBasePath + "orc_flow.zip"); + + //打包导出工程 + String exportPath = zipExportProject(orcExportSaveBasePath); + + //3、打包新的zip包上传BML + InputStream inputStream = bmlService.readLocalResourceFile(userName, exportPath); + Map resultMap = bmlService.upload(userName, inputStream, + dssOrchestratorInfo.getName() + ".OrcExport", projectName); + + //4、判断导出后是否改变Orc的版本 + if (addOrcVersion) { + Long orcIncreaseVersionId = orchestratorVersionIncrease(dssOrchestratorInfo.getId(), + userName, dssOrchestratorInfo.getComment(), + workspace, dssOrchestratorInfo, projectName, dssLabels); + resultMap.put("orcVersionId", orcIncreaseVersionId); + } else { + resultMap.put("orcVersionId", orcVersionId); + } + return resultMap; + //4、返回BML存储信息 + } else { + throw new DSSErrorException(90038, "该Orchestrator的版本号不存在,请检查版本号是否正确."); + } + } + + @Override + public Long orchestratorVersionIncrease(Long orcId, + String userName, + String comment, + Workspace workspace, + DSSOrchestratorInfo dssOrchestratorInfo, + String projectName, + List dssLabels) throws DSSErrorException { + //对于导出来说,json需替换 subflowID + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orcId,1); + // TODO: 2020/3/25 set updator(userID 修改为userName后) + dssOrchestratorVersion.setUpdateTime(new Date()); + Long oldOrcVersionId = dssOrchestratorVersion.getId(); + String oldOrcVersion = dssOrchestratorVersion.getVersion(); + dssOrchestratorVersion.setId(null); + // 如果是project发布 ,version都是01,如果是工作流发布,version + 1 + dssOrchestratorVersion.setVersion(OrchestratorUtils.increaseVersion(oldOrcVersion)); + + //发布的comment应该更新到上个版本,而当前最新版本的comment应该修改为release from.. + dssOrchestratorVersion.setComment(String.format("release from version %s", oldOrcVersion)); + //更新老版本的comment + DSSOrchestratorVersion updateCommentVersion = new DSSOrchestratorVersion(); + updateCommentVersion.setId(oldOrcVersionId); + String realComment = comment != null ? comment : "release comment"; + updateCommentVersion.setComment(realComment); + if(StringUtils.isNotBlank(userName)){ + updateCommentVersion.setUpdater(userName); + dssOrchestratorVersion.setUpdater(userName); + } + updateCommentVersion.setUpdateTime(new Date()); + orchestratorMapper.updateOrchestratorVersion(updateCommentVersion); + + //要求AppConn对应第三方应用拷贝一个新的app出来关联,如工作流,需要新建一个新的工作流进行关联。 + //1、生成上下文ContextId + String contextId = contextService.createContextID(workspace.getWorkspaceName(), projectName, dssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion(), userName); + dssOrchestratorVersion.setContextId(contextId); + LOGGER.info("Create a new ContextId {} for orchestration {} to increase version to {}.", contextId, orcId, dssOrchestratorVersion.getVersion()); + DSSOrchestrator dssOrchestrator = orchestratorManager.getOrCreateOrchestrator(userName, workspace.getWorkspaceName(), dssOrchestratorInfo.getType(), + dssLabels); + RefJobContentResponseRef responseRef = OrchestrationDevelopmentOperationUtils.tryOrchestrationOperation(dssOrchestratorInfo, dssOrchestrator, userName, + workspace, dssLabels, DevelopmentIntegrationStandard::getRefCRUDService, + developmentService -> ((RefCRUDService) developmentService).getRefCopyOperation(), + dssContextRequestRef -> dssContextRequestRef.setContextId(contextId), + projectRefRequestRef -> projectRefRequestRef.setProjectName(projectName).setRefProjectId(dssOrchestratorVersion.getProjectId()), + (developmentOperation, developmentRequestRef) -> { + CopyRequestRef requestRef = (CopyRequestRef) developmentRequestRef; + Map refJobContent = MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, dssOrchestratorVersion.getAppId(), + OrchestratorRefConstant.ORCHESTRATION_DESCRIPTION, dssOrchestratorVersion.getComment()); + requestRef.setNewVersion(dssOrchestratorVersion.getVersion()).setRefJobContent(refJobContent); + return ((RefCopyOperation) developmentOperation).copyRef(requestRef); + }, "increase"); + dssOrchestratorVersion.setAppId((Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY)); + dssOrchestratorVersion.setContent((String) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY)); + //update appConn node contextId + dssOrchestratorVersion.setFormatContextId(contextId); + orchestratorMapper.addOrchestratorVersion(dssOrchestratorVersion); + return dssOrchestratorVersion.getId(); + } + + @Override + public Long addVersionAfterPublish(String userName, Workspace workspace, Long orchestratorId, + Long orcVersionId, String projectName, + List dssLabels,String comment) throws DSSErrorException { + //发布之后添加一个版本号 + if (orcVersionId == null || orcVersionId < 0) { + LOGGER.info("orchestratorVersionId is {}", orcVersionId); + //最简单的就是通过orcId来找到最新的versionId + orcVersionId = orchestratorMapper.findLatestOrcVersionId(orchestratorId); + } + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getOrchestratorVersion(orcVersionId); + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(dssOrchestratorVersion.getOrchestratorId()); + Long orcIncreaseVersionId = orchestratorVersionIncrease(dssOrchestratorInfo.getId(), + userName, comment, + workspace, dssOrchestratorInfo, projectName, dssLabels); + orcIncreaseVersionId = orcIncreaseVersionId == null ? 0L : orcIncreaseVersionId; + return orcIncreaseVersionId; + } +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ImportDSSOrchestratorPluginImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ImportDSSOrchestratorPluginImpl.java new file mode 100644 index 000000000..ded19df02 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ImportDSSOrchestratorPluginImpl.java @@ -0,0 +1,214 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.impl; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.DSSLabelUtil; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestImportOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestProjectImportOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.AbstractDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.core.service.BMLService; +import com.webank.wedatasphere.dss.orchestrator.core.utils.OrchestratorUtils; +import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; +import com.webank.wedatasphere.dss.orchestrator.loader.OrchestratorManager; +import com.webank.wedatasphere.dss.orchestrator.publish.ImportDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.publish.io.input.MetaInputService; +import com.webank.wedatasphere.dss.orchestrator.publish.utils.OrchestrationDevelopmentOperationUtils; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.service.RefImportService; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.io.File; +import java.io.InputStream; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static com.webank.wedatasphere.dss.orchestrator.publish.impl.ExportDSSOrchestratorPluginImpl.DEFAULT_ORC_NAME; + + +@Component +public class ImportDSSOrchestratorPluginImpl extends AbstractDSSOrchestratorPlugin implements ImportDSSOrchestratorPlugin { + + @Autowired + private MetaInputService metaInputService; + @Autowired + private OrchestratorMapper orchestratorMapper; + @Autowired + private BMLService bmlService; + @Autowired + private ContextService contextService; + @Autowired + private OrchestratorManager orchestratorManager; + + @Override + @Transactional(rollbackFor = Exception.class) + public Long importOrchestrator(RequestImportOrchestrator requestImportOrchestrator) throws Exception { + String userName = requestImportOrchestrator.getUserName(); + String projectName = requestImportOrchestrator.getProjectName(); + Long projectId = requestImportOrchestrator.getProjectId(); + String resourceId = requestImportOrchestrator.getResourceId(); + String version = requestImportOrchestrator.getBmlVersion(); + List dssLabels = requestImportOrchestrator.getDssLabels(); + Workspace workspace = requestImportOrchestrator.getWorkspace(); + + //1、下载BML的Orchestrator的导入包 + String inputZipPath = IoUtils.generateIOPath(userName, projectName, DEFAULT_ORC_NAME + ".zip"); + bmlService.downloadToLocalPath(userName, resourceId, version, inputZipPath); + String inputPath = ZipHelper.unzip(inputZipPath); + + //2、导入Info信息(导入冲突处理) + List dssOrchestratorInfos = metaInputService.importOrchestrator(inputPath); + DSSOrchestratorInfo importDssOrchestratorInfo = dssOrchestratorInfos.get(0); + + //复制工程,直接使用新的UUID和复制后的工程ID + if (requestImportOrchestrator.getCopyProjectId() != null + && StringUtils.isNotBlank(requestImportOrchestrator.getCopyProjectName())) { + projectId = requestImportOrchestrator.getCopyProjectId(); + importDssOrchestratorInfo.setProjectId(projectId); + importDssOrchestratorInfo.setUUID(UUID.randomUUID().toString()); + } + //根据工程ID和编排名称查询 + String uuid = orchestratorMapper.getOrcNameByParam(importDssOrchestratorInfo.getProjectId(), importDssOrchestratorInfo.getName()); + //通过UUID查找是否已经导入过 + DSSOrchestratorInfo existFlag = orchestratorMapper.getOrchestratorByUUID(importDssOrchestratorInfo.getUUID()); + //add 和update都需要更新成当前环境id信息,放到新的版本记录中 + //todo 跨环境必须保证工程ID一样,或者需要更新导入包中的所有工程ID为当前最新ID,不一致的话关系到上下文、第三方工程的映射问题 + if (null != existFlag) { + //判断是否存在相同名称的编排名称 + if (StringUtils.isNotBlank(uuid) && !uuid.equals(importDssOrchestratorInfo.getUUID())) { + DSSExceptionUtils + .dealErrorException(61002, "The same orchestration name already exists", DSSErrorException.class); + } + importDssOrchestratorInfo.setId(existFlag.getId()); + //如果Orchestrator已经导入过,目前只更新版本信息,并更新基础信息name,其它信息不修改。 + orchestratorMapper.updateOrchestrator(importDssOrchestratorInfo); + } else { + //判断是否存在相同名称的编排名称 + if (StringUtils.isNotBlank(uuid)) { + DSSExceptionUtils + .dealErrorException(61002, "The same orchestration name already exists", DSSErrorException.class); + } + //使用生产环境的id + importDssOrchestratorInfo.setId(null); + importDssOrchestratorInfo.setCreator(userName); + importDssOrchestratorInfo.setCreateTime(new Date()); + //0.x工作流导入兼容 + if (importDssOrchestratorInfo.getWorkspaceId() == null) { + importDssOrchestratorInfo.setWorkspaceId(workspace.getWorkspaceId()); + } + if (StringUtils.isEmpty(importDssOrchestratorInfo.getOrchestratorMode())) { + importDssOrchestratorInfo.setOrchestratorMode("pom_work_flow"); + } + if (StringUtils.isEmpty(importDssOrchestratorInfo.getOrchestratorWay())) { + importDssOrchestratorInfo.setOrchestratorWay(",pom_work_flow_DAG,"); + } + orchestratorMapper.addOrchestrator(importDssOrchestratorInfo); + } + String flowZipPath = inputPath + File.separator + "orc_flow.zip"; + //3、上传工作流zip包到bml + InputStream inputStream = bmlService.readLocalResourceFile(userName, flowZipPath); + Map resultMap = bmlService.upload(userName, inputStream, importDssOrchestratorInfo.getName() + "_orc_flow.zip", projectName); + String orcResourceId = resultMap.get("resourceId").toString(); + String orcBmlVersion = resultMap.get("version").toString(); + + //4、导入版本Version信息 + DSSOrchestratorVersion dssOrchestratorVersion = new DSSOrchestratorVersion(); + dssOrchestratorVersion.setAppId(null); + dssOrchestratorVersion.setComment("orchestrator import"); + dssOrchestratorVersion.setOrchestratorId(importDssOrchestratorInfo.getId()); + dssOrchestratorVersion.setContent(""); + dssOrchestratorVersion.setProjectId(projectId); + dssOrchestratorVersion.setSource("Orchestrator create"); + dssOrchestratorVersion.setUpdater(userName); + //生产导入:默认是为无效,开发环境为有效 + dssOrchestratorVersion.setValidFlag(DSSLabelUtil.isDevEnv(dssLabels) ? 1 : 0); + + String oldVersion = orchestratorMapper.getLatestVersion(importDssOrchestratorInfo.getId(), 1); + if (StringUtils.isNotEmpty(oldVersion)) { + dssOrchestratorVersion.setVersion(OrchestratorUtils.increaseVersion(oldVersion)); + } else { + dssOrchestratorVersion.setVersion(OrchestratorUtils.generateNewVersion()); + } + + //5、生成上下文ContextId + String contextId = contextService.createContextID(workspace.getWorkspaceName(), projectName, importDssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion(), userName); + dssOrchestratorVersion.setFormatContextId(contextId); + LOGGER.info("Create a new ContextId for import: {} ", contextId); + + dssOrchestratorVersion.setUpdateTime(new Date()); + orchestratorMapper.addOrchestratorVersion(dssOrchestratorVersion); + + //6、导出第三方应用信息,如工作流、Visualis、Qualities + DSSOrchestrator dssOrchestrator = orchestratorManager.getOrCreateOrchestrator(userName, + workspace.getWorkspaceName(), importDssOrchestratorInfo.getType(), dssLabels); + Long finalProjectId = projectId; + RefJobContentResponseRef responseRef = OrchestrationDevelopmentOperationUtils.tryOrchestrationOperation(importDssOrchestratorInfo, + dssOrchestrator, userName, workspace, dssLabels, + DevelopmentIntegrationStandard::getRefImportService, + developmentService -> ((RefImportService) developmentService).getRefImportOperation(), + dssContextRequestRef -> dssContextRequestRef.setContextId(contextId), + projectRefRequestRef -> projectRefRequestRef.setRefProjectId(finalProjectId).setProjectName(projectName), + (developmentOperation, developmentRequestRef) -> { + ImportRequestRef requestRef = (ImportRequestRef) developmentRequestRef; + requestRef.setResourceMap(MapUtils.newCommonMap(ImportRequestRef.RESOURCE_ID_KEY, orcResourceId, ImportRequestRef.RESOURCE_VERSION_KEY, orcBmlVersion)); + requestRef.setNewVersion(dssOrchestratorVersion.getVersion()); + return ((RefImportOperation) developmentOperation).importRef(requestRef); + }, "import"); + long orchestrationId = (Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY); + String orchestrationContent = (String) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY); + + //更新返回內容 + dssOrchestratorVersion.setAppId(orchestrationId); + dssOrchestratorVersion.setContent(orchestrationContent); + orchestratorMapper.updateOrchestratorVersion(dssOrchestratorVersion); +// synProjectOrchestrator(importDssOrchestratorInfo, dssOrchestratorVersion, dssLabels); + return dssOrchestratorVersion.getOrchestratorId(); + } + + public void synProjectOrchestrator(DSSOrchestratorInfo importDssOrchestratorInfo, DSSOrchestratorVersion dssOrchestratorVersion, List dssLabels) { + //Is dev environment + if (DSSLabelUtil.isDevEnv(dssLabels)) { + RequestProjectImportOrchestrator projectImportOrchestrator = new RequestProjectImportOrchestrator(); + BeanUtils.copyProperties(importDssOrchestratorInfo, projectImportOrchestrator); + projectImportOrchestrator.setVersionId(dssOrchestratorVersion.getId()); + //保存工程级别的编排模式 + DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() + .ask(projectImportOrchestrator); + } + } +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaExportService.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaExportService.java new file mode 100644 index 000000000..c35c817eb --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaExportService.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.io.export; + + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; + +import java.io.IOException; + + +public interface MetaExportService { + + + /** + * + * @param dssOrchestratorInfo + * @param savePath + * @throws IOException + */ + void export(DSSOrchestratorInfo dssOrchestratorInfo, String savePath) throws IOException; + + +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaWriter.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaWriter.java new file mode 100644 index 000000000..1d7df1a39 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaWriter.java @@ -0,0 +1,173 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.io.export; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.orchestrator.publish.conf.DSSOrchestratorConf; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class MetaWriter { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final Class tClass; + private final String tableName; + private List datas = new ArrayList<>(); + private List ignoreFields = new ArrayList<>(); + private List fields = null; + private List comments = new ArrayList<>(); + private List table = new ArrayList<>(); + private String commentPrefix = "#"; + private String seperator = "|"; + + public MetaWriter(Class tClass, String tableName) { + this.tClass = tClass; + this.tableName = tableName; + } + + public static MetaWriter of(String tableName, Class tClass) { + return new MetaWriter<>(tClass, tableName); + } + + public MetaWriter ignore(String... fields) { + ignoreFields.addAll(Arrays.stream(fields).collect(Collectors.toList())); + return this; + } + + public MetaWriter data(List datas) { + this.datas.addAll(datas); + return this; + } + + public MetaWriter data(T data) { + this.datas.add(data); + return this; + } + + public MetaWriter comment(String... comments) { + this.comments.addAll(Arrays.stream(comments).collect(Collectors.toList())); + return this; + } + + /** + * 上传bml + * + * @return + */ + public InputStream write() { + writeComment(); + writeHead(); + writeBody(); + //table 添加换行符进行转流 + String tableStr = table.stream().reduce((a, b) -> a + "\n" + b).orElse("") + "\n"; + logger.info("\n" + tableStr); + return new ByteArrayInputStream(tableStr.getBytes()); + } + + /** + * 写入本地 + * + * @param outputStream + */ + public void write(OutputStream outputStream) throws IOException { + writeComment(); + writeHead(); + writeBody(); + IOUtils.writeLines(table, "\n", outputStream); + } + + private void writeBody() { + datas.forEach(DSSExceptionUtils.handling(this::writeBody)); + } + + private void writeBody(T t) throws NoSuchFieldException, IllegalAccessException { + ArrayList line = new ArrayList<>(); + for (String field : fields) { + Field declaredField = tClass.getDeclaredField(field); + declaredField.setAccessible(true); + Object o = declaredField.get(t); + if (o != null) { + line.add(o.toString()); + } else { + line.add(null); + } + } + //防止出现描述内容带有换行符,导致发布时导出的内容格式错乱 + table.add(reduce(line).replaceAll("[\n\r]"," ")); + } + + private String reduce(List strs) { + return strs.stream().reduce((a, b) -> a + seperator + b).orElse(""); + } + + /** + * 写表的头部 + * 1.过滤ignore的属性 + * 2.驼峰转mysql的 _ (暂时略过) + * 3.写入List table + */ + private void writeHead() { + fields = Arrays.stream(tClass.getDeclaredFields()) + .map(Field::getName) + .filter(n -> !ignoreFields.contains(n)) + .collect(Collectors.toList()); + table.add(reduce(fields)); + } + + /** + * 驼峰转_ + * + * @param str + * @return + */ + private String unCamel(String str) { + // TODO: 2020/3/9 + return null; + } + + /** + * 写comment,包括表名,class名,和外部自定义的comment + */ + private void writeComment() { + table.add(connectCommentPrefix(String.format("tableName:%s", tableName))); + table.add(connectCommentPrefix(String.format("class:%s", tClass.getName()))); + table.add(connectCommentPrefix(String.format("env:%s", DSSOrchestratorConf.DSS_EXPORT_ENV.getValue()))); + comments.stream().map(this::connectCommentPrefix).forEach(table::add); + } + + /** + * str前加上comment标识 + * + * @param str + * @return + */ + private String connectCommentPrefix(String str) { + return commentPrefix + str; + } + +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/impl/MetaExportServiceImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/impl/MetaExportServiceImpl.java new file mode 100644 index 000000000..929148faf --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/impl/MetaExportServiceImpl.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.io.export.impl; + + +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.publish.io.export.MetaExportService; +import com.webank.wedatasphere.dss.orchestrator.publish.io.export.MetaWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import java.io.*; + + +@Service +public class MetaExportServiceImpl implements MetaExportService { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + + private final String fileName = "meta.txt"; + + + @Override + public void export(DSSOrchestratorInfo dssOrchestratorInfo, String savePath) throws IOException { + + try ( + OutputStream outputStream = generateOutputStream(savePath) + ) { + exportOrchestratorBaseInfo(dssOrchestratorInfo, outputStream); + } + } + + + private OutputStream generateOutputStream(String basePath) throws IOException { + return IoUtils.generateExportOutputStream(basePath + File.separator + fileName); + } + + + private void exportOrchestratorBaseInfo(DSSOrchestratorInfo dssOrchestratorInfo,OutputStream outputStream) throws IOException { + + MetaWriter.of("dss_orchestrator", DSSOrchestratorInfo.class).data(dssOrchestratorInfo).write(outputStream); + + } + +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaInputService.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaInputService.java new file mode 100644 index 000000000..230ad50b7 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaInputService.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.io.input; + + + + + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; + +import java.io.IOException; +import java.util.List; + +public interface MetaInputService { + + + + List importOrchestrator(String basePath) throws IOException; + +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaReader.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaReader.java new file mode 100644 index 000000000..7b0854106 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaReader.java @@ -0,0 +1,166 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.io.input; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Field; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +public class MetaReader { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final Class tClass; + private final String tableName; + private String commentPrefix = "#"; + private String seperator = "\\|"; + private List datas = new ArrayList<>(); + private List fields = null; + private List comments = new ArrayList<>(); + private List> body = new ArrayList<>(); + private boolean finded = false; + private boolean firstLine = true; + + public MetaReader(Class tClass, String tableName) { + this.tClass = tClass; + this.tableName = tableName; + } + + public static MetaReader of(String tableName, Class tClass) { + return new MetaReader<>(tClass, tableName); + } + + public List read(InputStream inputStream) throws IOException { + try (InputStreamReader streamReader = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(streamReader);) { + readTable(reader); + } + readT(); + return datas; + } + + public String read(InputStream inputStream, String key) throws IOException { + try (InputStreamReader streamReader = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(streamReader);) { + readTable(reader); + } + String comment = comments.stream().filter(c -> c.contains(key + ":")).findFirst().orElse(""); + String[] split = comment.split(":"); + if (split.length > 1) return split[1]; + return ""; + } + + private void readT() { + body.stream().map(DSSExceptionUtils.map(this::lineToT)).forEach(datas::add); + } + + private T lineToT(List list) throws IllegalAccessException, InstantiationException, NoSuchFieldException, ParseException { + T t = tClass.newInstance(); + for (int i = 0; i < list.size(); i++) { + String valueStr = list.get(i); + if ("null".equalsIgnoreCase(valueStr)) continue; + Field declaredField = tClass.getDeclaredField(fields.get(i)); + declaredField.setAccessible(true); + Object value = null; + String type = declaredField.getType().getSimpleName(); + switch (type) { + case "String": + value = valueStr; + break; + case "Date": + value = new SimpleDateFormat("EEE MMM dd HH:mm:ss ZZZ yyyy", Locale.ENGLISH).parse(valueStr); + break; + case "Long": + case "long": + value = Long.valueOf(valueStr); + break; + case "Boolean": + value = Boolean.valueOf(valueStr); + break; + case "Integer": + value = Integer.valueOf(valueStr); + break; + default: + logger.warn(String.format("unsupport type %s", type)); + } + declaredField.set(t, value); + } + return t; + } + + private void readTable(BufferedReader reader) throws IOException { + String line = null; + while ((line = reader.readLine()) != null) { + if (!finded && !shut(line)) { + continue; + } + if (finded && isTableName(line)) { + break; + } + if (shut(line)) { + finded = true; + continue; + } + // TODO: 2020/3/9 + if (isComment(line)) { + comments.add(line); + continue; + } + if (firstLine) { + //handle head + fields = Arrays.stream(line.split(seperator)).collect(Collectors.toList()); + firstLine = false; + continue; + } + body.add(Arrays.stream(line.split(seperator)).collect(Collectors.toList())); + //handle body + } + } + + private boolean isComment(String str) { + return str.startsWith(commentPrefix); + } + + private boolean isTableName(String str) { + return isComment(str) && str.contains(commentPrefix + "tableName:"); + } + + private boolean isClass(String str) { + return isComment(str) && str.contains(commentPrefix + "class:"); + } + + private String getComment(String str) { + return str.substring(1); + } + + private boolean shut(String str) { + return isTableName(str) && getComment(str).equals(String.format("tableName:%s", tableName)); + } +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/impl/MetaInputServiceImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/impl/MetaInputServiceImpl.java new file mode 100644 index 000000000..52b9babbc --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/impl/MetaInputServiceImpl.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.io.input.impl; + + +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.publish.io.input.MetaInputService; +import com.webank.wedatasphere.dss.orchestrator.publish.io.input.MetaReader; + +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + + +@Service +public class MetaInputServiceImpl implements MetaInputService { + + // TODO: 2020/3/13 防止表结构发生改变的version 字段的添加 + + private final String fileName = "meta.txt"; + + @Override + public List importOrchestrator(String basePath) throws IOException { + try (InputStream inputStream = generateInputstream(basePath)) { + return MetaReader.of("dss_orchestrator", DSSOrchestratorInfo.class).read(inputStream); + } + } + + + /** + * 获取inputStream + * + * @param basePath + * @return + * @throws FileNotFoundException + */ + private InputStream generateInputstream(String basePath) throws IOException { + return IoUtils.generateInputInputStream(basePath + File.separator + fileName); + } + + +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/ConversionJobEntity.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/ConversionJobEntity.java new file mode 100644 index 000000000..7ef87563a --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/ConversionJobEntity.java @@ -0,0 +1,125 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.job; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.Date; +import java.util.List; +import java.util.Map; + + +public class ConversionJobEntity { + + private DSSProject project; + + //这里的 key 为 DSS 具体编排(如 DSS 工作流)的 id;这里的 value 为 DSS 编排所对应的第三方调度系统的工作流 ID + //请注意:由于对接的 SchedulerAppConn 调度系统有可能没有实现 OrchestrationService, + //所以可能存在 DSS 在创建 DSS 编排时,无法同步去 SchedulerAppConn 创建工作流的情况,从而导致这个 Map 的所有 value 都为 null。 + private Map orchestrationIdMap; + + // DSS编排的ID列表 + private List orcIdList; + + private Workspace workspace; + + private String userName; + + private List labels; + + private Date createTime; + + private Date updateTime; + + private ResponseOperateOrchestrator response; + + public DSSProject getProject() { + return project; + } + + public void setProject(DSSProject project) { + this.project = project; + } + + public Map getOrchestrationIdMap() { + return orchestrationIdMap; + } + + public void setOrchestrationIdMap(Map orchestrationIdMap) { + this.orchestrationIdMap = orchestrationIdMap; + } + + public List getOrcIdList() { + return orcIdList; + } + + public void setOrcIdList(List orcIdList) { + this.orcIdList = orcIdList; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public ResponseOperateOrchestrator getResponse() { + return response; + } + + public void setResponse(ResponseOperateOrchestrator response) { + updateTime = new Date(); + this.response = response; + } + + public List getLabels() { + return labels; + } + + public void setLabels(List labels) { + this.labels = labels; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/OrchestratorConversionJob.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/OrchestratorConversionJob.java new file mode 100644 index 000000000..364894274 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/OrchestratorConversionJob.java @@ -0,0 +1,108 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.publish.job; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.orchestrator.publish.ConversionDSSOrchestratorPlugin; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.linkis.common.utils.ByteTimeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.function.Consumer; + + +public final class OrchestratorConversionJob implements Runnable { + + private static final Logger LOGGER = LoggerFactory.getLogger(OrchestratorConversionJob.class); + + private String id; + + private ConversionJobEntity conversionJobEntity; + private List conversionDSSOrchestratorPlugins; + private Consumer consumer; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public void setConversionJobEntity(ConversionJobEntity conversionJobEntity) { + this.conversionJobEntity = conversionJobEntity; + } + + public ConversionJobEntity getConversionJobEntity() { + return conversionJobEntity; + } + + public void setConversionDSSOrchestratorPlugins(List conversionDSSOrchestratorPlugins) { + this.conversionDSSOrchestratorPlugins = conversionDSSOrchestratorPlugins; + } + + public void afterConversion(Consumer consumer) { + this.consumer = consumer; + } + + @Override + public void run() { + //1.从编排中心导出一次工作流,进行一次版本升级 + //2.进行发布到schedulis等调度系统 + LOGGER.info("Job {} begin to convert project {} for user {} to scheduler, the orchestrationIds is {}.", id, + conversionJobEntity.getProject().getId(), conversionJobEntity.getUserName(), conversionJobEntity.getOrchestrationIdMap().keySet()); + conversionJobEntity.setResponse(ResponseOperateOrchestrator.running()); + ConversionDSSOrchestratorPlugin conversionDSSOrchestratorPlugin = null; + for (DSSOrchestratorPlugin plugin: conversionDSSOrchestratorPlugins) { + if(plugin instanceof ConversionDSSOrchestratorPlugin) { + conversionDSSOrchestratorPlugin = (ConversionDSSOrchestratorPlugin) plugin; + } + } + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(conversionJobEntity.getProject().getId()); + try{ + DSSProject project = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() + .ask(projectInfoRequest); + conversionJobEntity.setProject(project); + Workspace workspace = conversionJobEntity.getWorkspace(); + ResponseOperateOrchestrator response = conversionDSSOrchestratorPlugin.convert(conversionJobEntity.getUserName(), project, workspace, + conversionJobEntity.getOrchestrationIdMap(), conversionJobEntity.getLabels()); + if(response.isFailed()) { + String msg = response.getMessage() == null ? "Unknown reason, please ask admin for help!" : response.getMessage(); + throw new DSSErrorException(50000, msg); + } + //3.如果都没有报错,那么默认任务应该是成功的,那么则将所有的状态进行置为完成 + consumer.accept(response); + conversionJobEntity.setResponse(response); + } catch (final Exception t){ + LOGGER.error("Job {} convert for project {} failed.", id, conversionJobEntity.getProject().getId(), t); + ResponseOperateOrchestrator response = ResponseOperateOrchestrator.failed(ExceptionUtils.getRootCauseMessage(t)); + conversionJobEntity.setResponse(response); + consumer.accept(response); + } + LOGGER.info("Job {} convert project {} for user {} to Orchestrator {}, costs {}.", id, conversionJobEntity.getProject().getId(), + conversionJobEntity.getUserName(), conversionJobEntity.getResponse().getJobStatus(), ByteTimeUtils.msDurationToString(conversionJobEntity.getUpdateTime().getTime() - conversionJobEntity.getCreateTime().getTime())); + } +} diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/utils/OrchestrationDevelopmentOperationUtils.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/utils/OrchestrationDevelopmentOperationUtils.java new file mode 100644 index 000000000..f9e3f3220 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/utils/OrchestrationDevelopmentOperationUtils.java @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.dss.orchestrator.publish.utils; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.loader.utils.OrchestratorLoaderUtils; +import com.webank.wedatasphere.dss.standard.app.development.operation.DevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.DSSContextRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.DevelopmentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.ProjectRefRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.development.utils.DevelopmentOperationUtils; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import org.apache.linkis.protocol.util.ImmutablePair; + +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * @author enjoyyin + * @date 2022-03-12 + * @since 0.5.0 + */ +public class OrchestrationDevelopmentOperationUtils { + + public static V tryOrchestrationOperation(DSSOrchestratorInfo dssOrchestratorInfo, + DSSOrchestrator dssOrchestrator, + String userName, + Workspace workspace, + List dssLabels, + BiFunction getDevelopmentService, + Function getDevelopmentOperation, + Consumer contextRequestRefConsumer, + Consumer projectRefRequestRefConsumer, + BiFunction responseRefConsumer, + String operationName) throws DSSErrorException { + ImmutablePair standMap = + OrchestratorLoaderUtils.getOrchestratorDevelopmentStandard(dssOrchestrator, dssLabels); + return DevelopmentOperationUtils.tryDevelopmentRequestRefOperation(() -> getDevelopmentService.apply(standMap.getValue(), standMap.getKey()), + getDevelopmentOperation, contextRequestRefConsumer, projectRefRequestRefConsumer, + (developmentOperation, developmentRequestRef) -> { + developmentRequestRef.setWorkspace(workspace).setUserName(userName).setDSSLabels(dssLabels).setType(dssOrchestratorInfo.getType()); + return responseRefConsumer.apply(developmentOperation, (K) developmentRequestRef); + }, + "Orchestrator type " + dssOrchestratorInfo.getType() + " with binding AppConn " + + dssOrchestratorInfo.getAppConnName() + " try to " + operationName); + } + +} diff --git a/dss-framework/pom.xml b/dss-framework/pom.xml new file mode 100644 index 000000000..902ad17ba --- /dev/null +++ b/dss-framework/pom.xml @@ -0,0 +1,42 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-framework + pom + + + dss-framework-common + dss-appconn-framework + dss-framework-workspace-server + framework-plugins/dss-framework-orchestrator-publish + dss-framework-orchestrator-server + dss-framework-project-server + dss-framework-admin-service + framework-plugins/dss-framework-migrate-server + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-common/pom.xml b/dss-orchestrator/dss-orchestrator-common/pom.xml new file mode 100644 index 000000000..4e31995b5 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/pom.xml @@ -0,0 +1,48 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-orchestrator-common + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestration.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestration.java new file mode 100644 index 000000000..7fa625b75 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestration.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.entity; + + +public interface DSSOrchestration { + + Long getId(); + + String getName(); + + String getDescription(); + +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java new file mode 100644 index 000000000..97233b66e --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java @@ -0,0 +1,267 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +import java.util.Date; +import java.util.List; + +public class DSSOrchestratorInfo implements DSSOrchestration { + + private Long id; + + private String name; + + private String type; + + private String desc; + + private String creator; + + private Date createTime; + + private String uses; + + private String appConnName; + + private Long projectId; + + private String uuid; + + private String secondaryType; + + private Long workspaceId; + + private String orchestratorMode; + + private String orchestratorWay; + + private String orchestratorLevel; + + private String updateUser; + + private Date updateTime; + + public DSSOrchestratorInfo() { + + } + + public DSSOrchestratorInfo(String name, String type, + String desc, String creator, + Date createTime, String uses, + String appConnName, Long projectId, String secondaryType, + List linkedAppConnNames, String comment) { + this.name = name; + this.type = type; + this.desc = desc; + this.creator = creator; + this.createTime = createTime; + this.uses = uses; + this.appConnName = appConnName; + this.projectId = projectId; + this.secondaryType = secondaryType; + this.linkedAppConnNames = linkedAppConnNames; + this.comment = comment; + } + + public String getSecondaryType() { + return secondaryType; + } + + public void setSecondaryType(String secondaryType) { + this.secondaryType = secondaryType; + } + + + public String getUUID() { + return uuid; + } + + public void setUUID(String uuid) { + this.uuid = uuid; + } + + + /** + * 用来存储该编排可以支持的节点类型名称 + */ + private List linkedAppConnNames; + + public List getLinkedAppConnNames() { + return linkedAppConnNames; + } + + public void setLinkedAppConnNames(List linkedAppConnNames) { + this.linkedAppConnNames = linkedAppConnNames; + } + + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + private String comment; + + public String getAppConnName() { + return appConnName; + } + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + @Override + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Override + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getDesc() { + return desc; + } + + @Override + public String getDescription() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public long getProjectId() { + return projectId; + } + + public void setProjectId(long projectId) { + this.projectId = projectId; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getOrchestratorMode() { + return orchestratorMode; + } + + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } + + public String getOrchestratorWay() { + return orchestratorWay; + } + + public void setOrchestratorWay(String orchestratorWay) { + this.orchestratorWay = orchestratorWay; + } + + public String getOrchestratorLevel() { + return orchestratorLevel; + } + + public void setOrchestratorLevel(String orchestratorLevel) { + this.orchestratorLevel = orchestratorLevel; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "DSSOrchestratorInfo{" + + "id=" + id + + ", name='" + name + '\'' + + ", type='" + type + '\'' + + ", desc='" + desc + '\'' + + ", creator='" + creator + '\'' + + ", createTime=" + createTime + + ", uses='" + uses + '\'' + + ", appConnName='" + appConnName + '\'' + + ", projectId=" + projectId + + ", uuid='" + uuid + '\'' + + ", secondaryType='" + secondaryType + '\'' + + ", linkedAppConnNames=" + linkedAppConnNames + + ", comment='" + comment + '\'' + + '}'; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorRefOrchestration.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorRefOrchestration.java new file mode 100644 index 000000000..600a9ca8b --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorRefOrchestration.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +public class DSSOrchestratorRefOrchestration { + + private Long id; + private Long orchestratorId; + private Long refProjectId; + private Long refOrchestrationId; + + public DSSOrchestratorRefOrchestration(Long orchestratorId, Long refProjectId, Long refOrchestrationId) { + this.orchestratorId = orchestratorId; + this.refProjectId = refProjectId; + this.refOrchestrationId = refOrchestrationId; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Long getRefProjectId() { + return refProjectId; + } + + public void setRefProjectId(Long refProjectId) { + this.refProjectId = refProjectId; + } + + public Long getRefOrchestrationId() { + return refOrchestrationId; + } + + public void setRefOrchestrationId(Long refOrchestrationId) { + this.refOrchestrationId = refOrchestrationId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java new file mode 100644 index 000000000..a27b26d40 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java @@ -0,0 +1,178 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.util.Date; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class DSSOrchestratorVersion { + private final static Logger LOGGER = LoggerFactory.getLogger(DSSOrchestratorVersion.class); + + + private Long id; + private Long orchestratorId; + private Long appId; + private Long projectId; + private String source; + private String version; + private String comment; + private Date updateTime; + private String updater; + private String content; + private String contextId; + //有效标示 0:无效;1:有效,默认是有效 + private Integer validFlag; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Long getAppId() { + return appId; + } + + public void setAppId(Long appId) { + this.appId = appId; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getUpdater() { + return updater; + } + + public void setUpdater(String updater) { + this.updater = updater; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public String getContextId() { + return contextId; + } + + public void setContextId(String contextId) { + this.contextId = contextId; + } + + public Integer getValidFlag() { + return validFlag; + } + + public void setValidFlag(Integer validFlag) { + this.validFlag = validFlag; + } + + /** + * 提取出contextId + */ + public void setFormatContextId(String contextId) { + try { + if (contextId == null || "".equals(contextId.trim()) || !contextId.contains("value") || !contextId.contains("contextId")) { + this.contextId = contextId; + } else { + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(contextId).getAsJsonObject(); + String tempValue = jsonObject.get("value").getAsString(); + String tempContextId = parser.parse(tempValue).getAsJsonObject().get("contextId").getAsString(); + this.contextId = tempContextId; + } + } catch (Exception e) { + LOGGER.error("setFormatContextIdError-", e); + this.contextId = contextId; + } + } + + @Override + public String toString() { + return "DSSOrchestratorVersion{" + + "id=" + id + + ", orchestratorId=" + orchestratorId + + ", appId=" + appId + + ", projectId=" + projectId + + ", source='" + source + '\'' + + ", version='" + version + '\'' + + ", comment='" + comment + '\'' + + ", updateTime=" + updateTime + + ", updater='" + updater + '\'' + + ", content='" + content + '\'' + + ", contextId='" + contextId + '\'' + + ", validFlag='" + validFlag + '\'' + + '}'; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorInfo.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorInfo.java new file mode 100644 index 000000000..2382c17d6 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorInfo.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.entity; + + +public class OrchestratorInfo { + + private Long orchestratorId; + + private Long orchestratorVersionId; + + public OrchestratorInfo(Long orchestratorId, Long orchestratorVersionId) { + this.orchestratorId = orchestratorId; + this.orchestratorVersionId = orchestratorVersionId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Long getOrchestratorVersionId() { + return orchestratorVersionId; + } + + public void setOrchestratorVersionId(Long orchestratorVersionId) { + this.orchestratorVersionId = orchestratorVersionId; + } + +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorUser.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorUser.java new file mode 100644 index 000000000..925016bab --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorUser.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +import java.util.Date; + + +public class OrchestratorUser { + + private Long id; + + private Long workspaceId; + + private Long projectId; + + private Long orchestratorId; + + private String username; + + //权限等级:0:私密;1:公开 + private Integer priv; + + private Date lastUpdateTime; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Integer getPriv() { + return priv; + } + + public void setPriv(Integer priv) { + this.priv = priv; + } + + public Date getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(Date lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorVo.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorVo.java new file mode 100644 index 000000000..c830188d3 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorVo.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +public class OrchestratorVo { + + DSSOrchestratorInfo dssOrchestratorInfo; + + DSSOrchestratorVersion dssOrchestratorVersion; + + public DSSOrchestratorInfo getDssOrchestratorInfo() { + return dssOrchestratorInfo; + } + + public void setDssOrchestratorInfo(DSSOrchestratorInfo dssOrchestratorInfo) { + this.dssOrchestratorInfo = dssOrchestratorInfo; + } + + public DSSOrchestratorVersion getDssOrchestratorVersion() { + return dssOrchestratorVersion; + } + + public void setDssOrchestratorVersion(DSSOrchestratorVersion dssOrchestratorVersion) { + this.dssOrchestratorVersion = dssOrchestratorVersion; + } + +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestAddVersionAfterPublish.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestAddVersionAfterPublish.java new file mode 100644 index 000000000..de7f9e33a --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestAddVersionAfterPublish.java @@ -0,0 +1,106 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.io.Serializable; +import java.util.List; + +public class RequestAddVersionAfterPublish implements Serializable { + + private String userName; + private Workspace workspace; + private Long orchestratorId; + private Long orcVersionId; + private String projectName; + private List dssLabels; + //发布备注 + private String comment; + + public RequestAddVersionAfterPublish(String userName, + Workspace workspace, + Long orchestratorId, + Long orcVersionId, + String projectName, + List dssLabels, String comment) { + this.userName = userName; + this.workspace = workspace; + this.orcVersionId = orcVersionId; + this.orchestratorId = orchestratorId; + this.projectName = projectName; + this.dssLabels = dssLabels; + this.comment = comment; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public Long getOrcVersionId() { + return orcVersionId; + } + + public void setOrcVersionId(Long orcVersionId) { + this.orcVersionId = orcVersionId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestConvertOrchestrations.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestConvertOrchestrations.java new file mode 100644 index 000000000..50ad403c0 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestConvertOrchestrations.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.entity.DSSWorkspace; +import com.webank.wedatasphere.dss.common.entity.project.Project; +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; +import java.util.Map; + + +public class RequestConvertOrchestrations { + + private String userName; + // 这里的 key 为 DSS 具体编排(如 DSS 工作流)的 id,value为SchedulerAppConn的工作流Id + private Map orchestrationIdMap; + private Project project; + private DSSWorkspace workspace; + private List dssLabels; + + public Map getOrchestrationIdMap() { + return orchestrationIdMap; + } + + public void setOrchestrationIdMap(Map orchestrationIdMap) { + this.orchestrationIdMap = orchestrationIdMap; + } + + public Project getProject() { + return project; + } + + public void setProject(Project project) { + this.project = project; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public DSSWorkspace getWorkspace() { + return workspace; + } + + public void setWorkspace(DSSWorkspace workspace) { + this.workspace = workspace; + } + + public List getDSSLabels() { + return dssLabels; + } + + public void setDSSLabels(List dssLabels) { + this.dssLabels = dssLabels; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestExportOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestExportOrchestrator.java new file mode 100644 index 000000000..30f177365 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestExportOrchestrator.java @@ -0,0 +1,106 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; + + +public class RequestExportOrchestrator { + + private String userName; + private Long orchestratorId; + private Long orcVersionId; + private String projectName; + private List dssLabels; + private Boolean addOrcVersion; + private Workspace workspace; + + public RequestExportOrchestrator(String userName, + Long orchestratorId, + Long orcVersionId, + String projectName, + List dssLabels, + Boolean addOrcVersion, + Workspace workspace) { + this.userName = userName; + this.orcVersionId = orcVersionId; + this.orchestratorId = orchestratorId; + this.projectName = projectName; + this.dssLabels = dssLabels; + this.addOrcVersion = addOrcVersion; + this.workspace = workspace; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Long getOrcVersionId() { + return orcVersionId; + } + + public void setOrcVersionId(Long orcVersionId) { + this.orcVersionId = orcVersionId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public Boolean getAddOrcVersion() { + return addOrcVersion; + } + + public void setAddOrcVersion(Boolean addOrcVersion) { + this.addOrcVersion = addOrcVersion; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestFrameworkConvertOrchestration.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestFrameworkConvertOrchestration.java new file mode 100644 index 000000000..310c283c9 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestFrameworkConvertOrchestration.java @@ -0,0 +1,91 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.entity.DSSWorkspace; +import java.util.List; +import java.util.Map; + + +public class RequestFrameworkConvertOrchestration { + + private String userName; + private DSSWorkspace workspace; + private Long orcAppId; + private List orcIds; + private Map labels; + private boolean convertAllOrcs; + private String comment; + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public DSSWorkspace getWorkspace() { + return workspace; + } + + public void setWorkspace(DSSWorkspace workspace) { + this.workspace = workspace; + } + + public Long getOrcAppId() { + return orcAppId; + } + + public void setOrcAppId(Long orcAppId) { + this.orcAppId = orcAppId; + } + + public List getOrcIds() { + return orcIds; + } + + public void setOrcIds(List orcIds) { + this.orcIds = orcIds; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + + public boolean isConvertAllOrcs() { + return convertAllOrcs; + } + + public void setConvertAllOrcs(boolean convertAllOrcs) { + this.convertAllOrcs = convertAllOrcs; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestFrameworkConvertOrchestrationStatus.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestFrameworkConvertOrchestrationStatus.java new file mode 100644 index 000000000..8a85bba5f --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestFrameworkConvertOrchestrationStatus.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + + +public class RequestFrameworkConvertOrchestrationStatus { + + private String id; + + public RequestFrameworkConvertOrchestrationStatus() { + } + + public RequestFrameworkConvertOrchestrationStatus(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestImportOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestImportOrchestrator.java new file mode 100644 index 000000000..0784ee16e --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestImportOrchestrator.java @@ -0,0 +1,137 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; + + +public class RequestImportOrchestrator { + + + private String userName; + private String projectName; + private Long projectId; + private String resourceId; + private String bmlVersion; + private String orchestratorName; + private List dssLabels; + private Workspace workspace; + private Long copyProjectId; + private String copyProjectName; + + public RequestImportOrchestrator(String userName, + String projectName, + Long projectId, + String resourceId, + String bmlVersion, + String orchestratorName, + List dssLabels, + Workspace workspace) { + this.userName = userName; + this.workspace = workspace; + this.projectName = projectName; + this.projectId = projectId; + this.resourceId = resourceId; + this.bmlVersion = bmlVersion; + this.dssLabels = dssLabels; + this.orchestratorName = orchestratorName; + } + + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public Long getCopyProjectId() { + return copyProjectId; + } + + public void setCopyProjectId(Long copyProjectId) { + this.copyProjectId = copyProjectId; + } + + public String getCopyProjectName() { + return copyProjectName; + } + + public void setCopyProjectName(String copyProjectName) { + this.copyProjectName = copyProjectName; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorInfos.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorInfos.java new file mode 100644 index 000000000..2e2e4f123 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorInfos.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +public class RequestOrchestratorInfos { + private Long workspaceId; + private Long projectId; + private String name; + private String orchestratorMode; + + + public RequestOrchestratorInfos(Long workspaceId, Long projectId, String name, String orchestratorMode) { + this.workspaceId = workspaceId; + this.projectId = projectId; + this.name = name; + this.orchestratorMode = orchestratorMode; + } + + public RequestOrchestratorInfos() { + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getOrchestratorMode() { + return orchestratorMode; + } + + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorVersion.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorVersion.java new file mode 100644 index 000000000..d71a4d7f8 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorVersion.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + + +public class RequestOrchestratorVersion { + + private String username; + private Long projectId; + private Long orchestratorId; + private String dssLabel; + + + public static RequestOrchestratorVersion newInstance(String username, Long projectId, Long orchestratorId, String dssLabel) { + RequestOrchestratorVersion requestOrchestratorVersion = new RequestOrchestratorVersion(); + requestOrchestratorVersion.setOrchestratorId(orchestratorId); + requestOrchestratorVersion.setDssLabel(dssLabel); + requestOrchestratorVersion.setProjectId(projectId); + requestOrchestratorVersion.setUsername(username); + return requestOrchestratorVersion; + } + + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getDssLabel() { + return dssLabel; + } + + public void setDssLabel(String dssLabel) { + this.dssLabel = dssLabel; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestProjectImportOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestProjectImportOrchestrator.java new file mode 100644 index 000000000..db4d1be92 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestProjectImportOrchestrator.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; + +public class RequestProjectImportOrchestrator extends DSSOrchestratorInfo { + + private Long versionId; + + public Long getVersionId() { + return versionId; + } + + public void setVersionId(Long versionId) { + this.versionId = versionId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestProjectUpdateOrcVersion.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestProjectUpdateOrcVersion.java new file mode 100644 index 000000000..6f3928999 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestProjectUpdateOrcVersion.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; + +public class RequestProjectUpdateOrcVersion { + private Long projectId; + private Long orchestratorId; + private Long versionId; + private List dssLabels; + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Long getVersionId() { + return versionId; + } + + public void setVersionId(Long versionId) { + this.versionId = versionId; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQueryByIdOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQueryByIdOrchestrator.java new file mode 100644 index 000000000..b41759b33 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQueryByIdOrchestrator.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +/** + * Description + */ + +public class RequestQueryByIdOrchestrator { + private Long orchestratorId; + private Long orcVersionId; + + public RequestQueryByIdOrchestrator(Long orchestratorId, Long orcVersionId) { + this.orchestratorId = orchestratorId; + this.orcVersionId = orcVersionId; + } + + public RequestQueryByIdOrchestrator() { + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Long getOrcVersionId() { + return orcVersionId; + } + + public void setOrcVersionId(Long orcVersionId) { + this.orcVersionId = orcVersionId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQueryOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQueryOrchestrator.java new file mode 100644 index 000000000..04290298b --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQueryOrchestrator.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import java.util.List; + + +public class RequestQueryOrchestrator { + private List orchestratorIds; + + public RequestQueryOrchestrator(List orchestratorIds) { + this.orchestratorIds = orchestratorIds; + } + + public List getOrchestratorIds() { + return orchestratorIds; + } + + public void setOrchestratorIds(List orchestratorIds) { + this.orchestratorIds = orchestratorIds; + } + + +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuerySchedulerWorkflowStatus.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuerySchedulerWorkflowStatus.java new file mode 100644 index 000000000..62970c5aa --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuerySchedulerWorkflowStatus.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +public class RequestQuerySchedulerWorkflowStatus { + private String username; + private Long orchestratorId; + + public RequestQuerySchedulerWorkflowStatus(String username, Long orchestratorId) { + this.username = username; + this.orchestratorId = orchestratorId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseConvertOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseConvertOrchestrator.java new file mode 100644 index 000000000..111f1ddd1 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseConvertOrchestrator.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + + +public class ResponseConvertOrchestrator { + + private String id; + private ResponseOperateOrchestrator response; + + public ResponseConvertOrchestrator() { + } + + public ResponseConvertOrchestrator(String id, ResponseOperateOrchestrator response) { + this.id = id; + this.response = response; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public ResponseOperateOrchestrator getResponse() { + return response; + } + + public void setResponse(ResponseOperateOrchestrator response) { + this.response = response; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOperateOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOperateOrchestrator.java new file mode 100644 index 000000000..6cc64d230 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOperateOrchestrator.java @@ -0,0 +1,98 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.protocol.JobStatus; + + +public class ResponseOperateOrchestrator { + + private JobStatus jobStatus; + private String message; + + public JobStatus getJobStatus() { + return jobStatus; + } + + public ResponseOperateOrchestrator setJobStatus(JobStatus jobStatus) { + this.jobStatus = jobStatus; + return this; + } + + public String getMessage() { + return message; + } + + public ResponseOperateOrchestrator setMessage(String message) { + this.message = message; + return this; + } + + public boolean isCompleted() { + return isSucceed() || isFailed(); + } + + public boolean isSucceed() { + return jobStatus == JobStatus.Success; + } + + public boolean isFailed() { + return jobStatus == JobStatus.Failed; + } + + public boolean isRunning() { + return jobStatus == JobStatus.Running; + } + + public static ResponseOperateOrchestrator success() { + ResponseOperateOrchestrator response = new ResponseOperateOrchestrator(); + response.setJobStatus(JobStatus.Success); + return response; + } + + public static ResponseOperateOrchestrator success(String message) { + ResponseOperateOrchestrator response = new ResponseOperateOrchestrator(); + response.setJobStatus(JobStatus.Success); + response.setMessage(message); + return response; + } + + public static ResponseOperateOrchestrator failed() { + ResponseOperateOrchestrator response = new ResponseOperateOrchestrator(); + response.setJobStatus(JobStatus.Failed); + return response; + } + + public static ResponseOperateOrchestrator failed(String message) { + ResponseOperateOrchestrator response = new ResponseOperateOrchestrator(); + response.setJobStatus(JobStatus.Failed); + response.setMessage(message); + return response; + } + + public static ResponseOperateOrchestrator running() { + ResponseOperateOrchestrator response = new ResponseOperateOrchestrator(); + response.setJobStatus(JobStatus.Running); + return response; + } + + public static ResponseOperateOrchestrator inited() { + ResponseOperateOrchestrator response = new ResponseOperateOrchestrator(); + response.setJobStatus(JobStatus.Inited); + return response; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOrchestratorInfos.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOrchestratorInfos.java new file mode 100644 index 000000000..2a5822c98 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOrchestratorInfos.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; + +import java.util.List; + +public class ResponseOrchestratorInfos { + List orchestratorInfos; + + public ResponseOrchestratorInfos(List orchestratorInfos) { + this.orchestratorInfos = orchestratorInfos; + } + + public List getOrchestratorInfos() { + return orchestratorInfos; + } + + public void setOrchestratorInfos(List orchestratorInfos) { + this.orchestratorInfos = orchestratorInfos; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOrchetratorVersion.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOrchetratorVersion.java new file mode 100644 index 000000000..99f0f1478 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseOrchetratorVersion.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; + +import java.util.List; + + +public class ResponseOrchetratorVersion { + + private Long projectId; + private Long orchestratorId; + private List orchestratorVersions; + + public ResponseOrchetratorVersion(Long projectId, Long orchestratorId, List orchestratorVersions) { + this.projectId = projectId; + this.orchestratorId = orchestratorId; + this.orchestratorVersions = orchestratorVersions; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public List getOrchestratorVersions() { + return orchestratorVersions; + } + + public void setOrchestratorVersions(List orchestratorVersions) { + this.orchestratorVersions = orchestratorVersions; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseQueryOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseQueryOrchestrator.java new file mode 100644 index 000000000..31e70372f --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/ResponseQueryOrchestrator.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + + +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; + +import java.util.List; + + +public class ResponseQueryOrchestrator { + + private List orchestratorVoes; + + public ResponseQueryOrchestrator(List orchestratorVoes) { + this.orchestratorVoes = orchestratorVoes; + } + + public List getOrchestratorVoes() { + return orchestratorVoes; + } + + public void setOrchestratorVoes(List orchestratorVoes) { + this.orchestratorVoes = orchestratorVoes; + } + + +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java new file mode 100644 index 000000000..1b1f7df8f --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.orchestrator.common.ref; + +/** + * @author enjoyyin + * @date 2022-03-10 + * @since 0.5.0 + */ +public interface OrchestratorRefConstant { + + String DSS_ORCHESTRATOR_INFO_KEY = "dssOrchestratorInfo"; + + /*************************************************************/ + /** These keys below are used by Orchestrator Framework, **/ + /** so it is represented the orchestrator framework abstract defination. **/ + /*************************************************************/ + String ORCHESTRATOR_ID_KEY = "orchestratorId"; + String ORCHESTRATOR_VERSION_KEY = "orchestratorVersion"; + + + /*************************************************************/ + /** These keys below are used by the real implementations of Orchestrator, such as workflow, **/ + /** which is a kind of realized orchestrator of DSS. + * So it is represented the detail orchestrator. we use orchestration in head to distinct the difference of + * the keys of orchestrator **/ + /*************************************************************/ + String ORCHESTRATION_ID_KEY = "orchestrationId"; + String ORCHESTRATION_CONTENT_KEY = "orchestrationContent"; + String ORCHESTRATION_NAME = "orchestrationName"; + String ORCHESTRATION_DESCRIPTION = "orchestrationDescription"; + String ORCHESTRATION_USES = "orchestrationUses"; + String ORCHESTRATION_SCHEDULER_APP_CONN = "schedulerAppConnName"; + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml b/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml new file mode 100644 index 000000000..012de0c44 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml @@ -0,0 +1,72 @@ + + + + + + dss-orchestrator + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-orchestrator-conversion-standard + jar + + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + com.webank.wedatasphere.dss + dss-orchestrator-common + ${dss.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/AbstractConversionIntegrationStandard.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/AbstractConversionIntegrationStandard.java new file mode 100644 index 000000000..e4d77d37a --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/AbstractConversionIntegrationStandard.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.ConversionService; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.DSSToRelConversionService; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.RelToOrchestratorConversionService; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.core.AbstractAppIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; + +import java.io.IOException; + + +public abstract class AbstractConversionIntegrationStandard extends AbstractAppIntegrationStandard + implements ConversionIntegrationStandard { + + private AppConn appConn; + + protected abstract DSSToRelConversionService createDSSToRelConversionService(); + + /** + * 预留接口,用于支持将调度系统的工作流,转换成DSS编排 + * @return 目前返回 null 即可 + */ + protected RelToOrchestratorConversionService createRelToDSSConversionService() { + return null; + } + + @Override + public final DSSToRelConversionService getDSSToRelConversionService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createDSSToRelConversionService, DSSToRelConversionService.class); + } + + @Override + public final RelToOrchestratorConversionService getRelToDSSConversionService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createRelToDSSConversionService, RelToOrchestratorConversionService.class); + } + + @Override + protected void initService(T service) { + super.initService(service); + service.setAppStandard(this); + } + + @Override + public void init() throws AppStandardErrorException { + + } + + @Override + public AppConn getAppConn() { + return appConn; + } + + public void setAppConn(AppConn appConn) { + this.appConn = appConn; + } + + @Override + public String getAppConnName() { + return appConn.getAppDesc().getAppName(); + } + + @Override + public void close() throws IOException { + + } +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ConversionIntegrationStandard.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ConversionIntegrationStandard.java new file mode 100644 index 000000000..f9964069c --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ConversionIntegrationStandard.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.DSSToRelConversionService; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.RelToOrchestratorConversionService; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.core.AppIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + + +public interface ConversionIntegrationStandard extends AppIntegrationStandard { + + /** + * 用于支持将DSS编排,转换为调度系统的工作流 + * @return 返回具体实现类 + */ + DSSToRelConversionService getDSSToRelConversionService(AppInstance appInstance); + + /** + * 预留接口,用于支持将调度系统的工作流,转换成DSS编排 + * @return 目前返回 null 即可 + */ + RelToOrchestratorConversionService getRelToDSSConversionService(AppInstance appInstance); + + /** + * 由于 ConversionIntegration 是一个独立于三级规范之外的插件式规范,所以该规范没有途径可以拿到所需的 + * 三级规范中的一些Operation,这里预留一个接口,允许 ConversionIntegration 调用三级规范的 Operation. + * @return 用户实现的AppConn + */ + AppConn getAppConn(); + + String getAppConnName(); + + @Override + default String getStandardName() { + return "conversionIntegrationStandard"; + } + + @Override + default int getGrade() { + return 4; + } + + @Override + default boolean isNecessary() { + return false; + } +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/operation/ConversionOperation.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/operation/ConversionOperation.java new file mode 100644 index 000000000..5d3ebbbd2 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/operation/ConversionOperation.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.operation; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.ConversionRequestRef; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.ConversionService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.service.Operation; + + +public interface ConversionOperation extends Operation { + + void setConversionService(ConversionService conversionService); + + V convert(K ref); + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/operation/DSSToRelConversionOperation.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/operation/DSSToRelConversionOperation.java new file mode 100644 index 000000000..b8d4c06c9 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/operation/DSSToRelConversionOperation.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.operation; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.DSSToRelConversionRequestRef; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.ConversionService; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.DSSToRelConversionService; +import com.webank.wedatasphere.dss.standard.app.sso.operation.AbstractOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; + + +public abstract class DSSToRelConversionOperation, V extends ResponseRef> + extends AbstractOperation implements ConversionOperation { + + @Override + public final void setConversionService(ConversionService conversionService) { + this.service = conversionService; + } + + public DSSToRelConversionService getConversionService() { + return (DSSToRelConversionService) this.service; + } +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/ConversionRequestRef.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/ConversionRequestRef.java new file mode 100644 index 000000000..d0c85f8e5 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/ConversionRequestRef.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.ref; + +import com.webank.wedatasphere.dss.standard.app.sso.ref.WorkspaceRequestRef; + + +public interface ConversionRequestRef extends WorkspaceRequestRef { + + String getUserName(); + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/DSSToRelConversionRequestRef.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/DSSToRelConversionRequestRef.java new file mode 100644 index 000000000..113d69ed0 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/DSSToRelConversionRequestRef.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.ref; + + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +public interface DSSToRelConversionRequestRef> + extends ConversionRequestRef { + + R setUserName(String userName); + + R setWorkspace(Workspace workspace); + + default DSSProject getDSSProject() { + return (DSSProject) getParameter("dssProject"); + } + + default R setDSSProject(DSSProject dssProject) { + setParameter("dssProject", dssProject); + return (R) this; + } + + class ProjectToRelConversionRequestRefImpl extends DSSToRelConversionRequestRefImpl + implements ProjectToRelConversionRequestRef {} + + class OrchestrationToRelConversionRequestRefImpl extends DSSToRelConversionRequestRefImpl + implements OrchestrationToRelConversionRequestRef {} + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/DSSToRelConversionRequestRefImpl.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/DSSToRelConversionRequestRefImpl.java new file mode 100644 index 000000000..23b7e7464 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/DSSToRelConversionRequestRefImpl.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.dss.orchestrator.converter.standard.ref; + +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRefImpl; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public class DSSToRelConversionRequestRefImpl> + extends RequestRefImpl implements DSSToRelConversionRequestRef { + + private Workspace workspace; + private String userName; + + @Override + public String getUserName() { + return userName; + } + + @Override + public R setUserName(String userName) { + this.userName = userName; + return (R) this; + } + + @Override + public Workspace getWorkspace() { + return workspace; + } + + @Override + public R setWorkspace(Workspace workspace) { + this.workspace = workspace; + return (R) this; + } + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/OrchestrationToRelConversionRequestRef.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/OrchestrationToRelConversionRequestRef.java new file mode 100644 index 000000000..f2421fc10 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/OrchestrationToRelConversionRequestRef.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.dss.orchestrator.converter.standard.ref; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestration; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public interface OrchestrationToRelConversionRequestRef> + extends DSSToRelConversionRequestRef { + + default DSSOrchestration getDSSOrchestration() { + return (DSSOrchestration) getParameter("dssOrchestration"); + } + + default R setDSSOrchestration(DSSOrchestration dssOrchestration) { + setParameter("dssOrchestration", dssOrchestration); + return (R) this; + } + + /** + * 返回一个 SchedulerAppConn(调度系统)工作流的 id,也有可能返回 null。 + * 该调度系统的工作流 id,与 getDSSOrchestration 方法所获取到的 DSS 编排存在一对一的关系。 + * 如何判断是否为空?第三方 SchedulerAppConn 如果实现了 OrchestrationService,则必定会存在,否则为空。 + * 具体请参考 OrchestrationService 的类注释。 + * @return + */ + default Long getRefOrchestrationId() { + return (Long) getParameter("refOrchestrationId"); + } + + default R setRefOrchestrationId(Long refOrchestrationId) { + setParameter("refOrchestrationId", refOrchestrationId); + return (R) this; + } + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/ProjectToRelConversionRequestRef.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/ProjectToRelConversionRequestRef.java new file mode 100644 index 000000000..18e610bed --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/ref/ProjectToRelConversionRequestRef.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.ref; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestration; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import java.util.List; +import java.util.Map; + + +public interface ProjectToRelConversionRequestRef> + extends DSSToRelConversionRequestRef { + + default List getDSSOrcList() { + return (List) getParameter("dssOrcList"); + } + + default R setDSSOrcList(List dssOrcList) { + setParameter("dssOrcList", dssOrcList); + return (R) this; + } + + /** + * 该 Map 会返回 DSS 编排,与 SchedulerAppConn(调度系统)工作流的一一对应关系,也有可能返回 null。 + * 其中:key 为 DSS 编排名,value 为 SchedulerAppConn 工作流的 id。 + * 如何判断是否为空?第三方 SchedulerAppConn 如果实现了 OrchestrationService,则必定会存在,否则为空。 + * 具体请参考 OrchestrationService 的类注释。 + * @return 返回 DSS 编排与 SchedulerAppConn(调度系统)工作流的一一对应关系。 + */ + default Map getRefOrchestrationIdMap() { + return (Map) getParameter("refOrchestrationIdMap"); + } + + default R setRefOrchestrationId(Map refOrchestrationIdMap) { + setParameter("refOrchestrationIdMap", refOrchestrationIdMap); + return (R) this; + } + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/AbstractConversionService.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/AbstractConversionService.java new file mode 100644 index 000000000..1b88fd254 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/AbstractConversionService.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.service; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ConversionIntegrationStandard; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.ConversionOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppSingletonIntegrationServiceImpl; +import com.webank.wedatasphere.dss.standard.common.core.AppStandard; +import java.util.ArrayList; +import java.util.List; + + +public abstract class AbstractConversionService extends AppSingletonIntegrationServiceImpl + implements ConversionService { + + private List dssLabels = new ArrayList<>(); + private ConversionIntegrationStandard appStandard; + + public void setLabels(List labels) { + dssLabels = labels; + } + + @Override + public List getLabels() { + return dssLabels; + } + + @Override + protected void initOperation(ConversionOperation operation) { + operation.setConversionService(this); + } + + @Override + public ConversionIntegrationStandard getAppStandard() { + return appStandard; + } + + @Override + public void setAppStandard(ConversionIntegrationStandard appStandard) { + this.appStandard = appStandard; + } +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/ConversionService.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/ConversionService.java new file mode 100644 index 000000000..b03256595 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/ConversionService.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.service; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ConversionIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppIntegrationService; +import com.webank.wedatasphere.dss.standard.common.core.AppStandard; +import java.util.List; + + +public interface ConversionService extends AppIntegrationService { + + /** + * Labels by default. + * @return All default labels + */ + List getLabels(); + + void setAppStandard(ConversionIntegrationStandard appStandard); + + ConversionIntegrationStandard getAppStandard(); + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/DSSToRelConversionService.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/DSSToRelConversionService.java new file mode 100644 index 000000000..748b56d01 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/DSSToRelConversionService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.service; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.DSSToRelConversionOperation; + + +public interface DSSToRelConversionService extends ConversionService { + + DSSToRelConversionOperation getDSSToRelConversionOperation(); + + default boolean isConvertAllOrcs() { + return true; + } + +} diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/RelToOrchestratorConversionService.java b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/RelToOrchestratorConversionService.java new file mode 100644 index 000000000..9cff6afb0 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/src/main/java/com/webank/wedatasphere/dss/orchestrator/converter/standard/service/RelToOrchestratorConversionService.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.converter.standard.service; + + +public interface RelToOrchestratorConversionService extends ConversionService { + +} diff --git a/dss-orchestrator/dss-orchestrator-core/pom.xml b/dss-orchestrator/dss-orchestrator-core/pom.xml new file mode 100644 index 000000000..64cf71080 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/pom.xml @@ -0,0 +1,116 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-orchestrator-core + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-appconn-core + ${dss.version} + + + + linkis-bml-client + + + gson + com.google.code.gson + + + org.apache.linkis + ${linkis.version} + + + + com.webank.wedatasphere.dss + dss-contextservice + ${dss.version} + + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-orchestrator-common + ${dss.version} + compile + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.xml + **/*.properties + **/*.yml + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/DSSOrchestrator.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/DSSOrchestrator.java new file mode 100644 index 000000000..2060913a4 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/DSSOrchestrator.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; + +import java.util.List; + + + +public interface DSSOrchestrator { + + /** + * 返回Orchestrator的名称,如workflow + * @return + */ + String getName(); + + + /** + * 返回编排关联的AppConn + * @return + */ + AppConn getAppConn(); + + /** + * 返回编排关联的 SchedulerAppConn。 + * DSSOrchestrator 允许每个编排绑定一个特有的 SchedulerAppConn。当 orchestrator-framework 在管理一个 + * Orchestrator 时(即增删改查一个 DSSOrchestrator),会同步向 SchedulerAppConn 请求,希望能够在关联的 + * 调度系统也进行这样的操作。 + *
+ * 一般情况下,整个 DSS 会至少加载一个 SchedulerAppConn,如果 DSSOrchestrator 没有显示的绑定一个 SchedulerAppConn, + * 则 DSS 系统会默认为该 DSSOrchestrator 随机绑定一个。 + * @return 返回绑定的 SchedulerAppConn + */ + SchedulerAppConn getSchedulerAppConn(); + + DSSOrchestratorContext getDSSOrchestratorContext(); + + /** + * 返回所有已经关联到的AppConn + * @return + */ + List getLinkedAppConn(); + + /** + * 用于工具条功能按钮展示,可以查到该模式可以提供的功能按钮 + * @return 按钮列表 + */ + List getToolBars(); +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/DSSOrchestratorContext.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/DSSOrchestratorContext.java new file mode 100644 index 000000000..bef0e6cf9 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/DSSOrchestratorContext.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core; + +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; +import java.io.Closeable; +import java.util.List; +import java.util.Map; + + +public interface DSSOrchestratorContext extends Closeable { + + void initialize(); + + Map getConfigMap(); + + List getOrchestratorPlugins(); + + default T getDSSOrchestratorPlugin(Class clazz) { + return (T) getOrchestratorPlugins().stream().filter(clazz::isInstance) + .findFirst().orElseThrow(() -> new DSSRuntimeException(50321, "Cannot find " + clazz.getSimpleName())); + } + + boolean isActive(); + +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/exception/DSSOrchestratorErrorException.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/exception/DSSOrchestratorErrorException.java new file mode 100644 index 000000000..601ca0074 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/exception/DSSOrchestratorErrorException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + + +public class DSSOrchestratorErrorException extends DSSErrorException { + public DSSOrchestratorErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public DSSOrchestratorErrorException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/AbstractDSSOrchestratorContext.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/AbstractDSSOrchestratorContext.java new file mode 100644 index 000000000..b2f1799d3 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/AbstractDSSOrchestratorContext.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.impl; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; +import java.util.HashMap; +import java.util.Map; + + +public abstract class AbstractDSSOrchestratorContext implements DSSOrchestratorContext { + + private boolean isClosed = false; + private Map configMap = new HashMap<>(); + + + @Override + public Map getConfigMap() { + return configMap; + } + + @Override + public boolean isActive() { + return !isClosed; + } + + @Override + public void close() { + isClosed = true; + getOrchestratorPlugins().forEach(DSSExceptionUtils.handling(DSSOrchestratorPlugin::close)); + } +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/AbstractOrchestrator.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/AbstractOrchestrator.java new file mode 100644 index 000000000..f11c5f1ee --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/AbstractOrchestrator.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.impl; + +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; + +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext; +import java.util.Arrays; +import java.util.List; + + +abstract class AbstractOrchestrator implements DSSOrchestrator { + + private volatile DSSOrchestratorContext dssOrchestratorContext; + + @Override + public List getToolBars() { + String[] toolNames = {"参数", "资源", "执行", "发布","保存"}; + return Arrays.asList(toolNames); + } + + protected abstract DSSOrchestratorContext createOrchestratorContext(); + + @Override + public DSSOrchestratorContext getDSSOrchestratorContext() { + if(dssOrchestratorContext == null) { + synchronized (this) { + if(dssOrchestratorContext == null) { + dssOrchestratorContext = createOrchestratorContext(); + } + } + } + return dssOrchestratorContext; + } +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/DSSOrchestratorContextImpl.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/DSSOrchestratorContextImpl.java new file mode 100644 index 000000000..6cc3f940d --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/DSSOrchestratorContextImpl.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.impl; + +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; +import java.util.List; + + +public class DSSOrchestratorContextImpl extends AbstractDSSOrchestratorContext { + + private List plugins; + + @Override + public void initialize() { + plugins = ClassUtils.getInstances(DSSOrchestratorPlugin.class); + plugins.forEach(DSSOrchestratorPlugin::init); + } + + @Override + public List getOrchestratorPlugins() { + return plugins; + } + +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/DefaultOrchestrator.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/DefaultOrchestrator.java new file mode 100644 index 000000000..ccf108a85 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/impl/DefaultOrchestrator.java @@ -0,0 +1,106 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext; + +import java.util.ArrayList; +import java.util.List; + + +public class DefaultOrchestrator extends AbstractOrchestrator { + + private static volatile DSSOrchestratorContext orchestratorContext; + + private static void initDSSOrchestratorContext() { + if(orchestratorContext == null) { + synchronized (DefaultOrchestrator.class) { + if(orchestratorContext == null) { + orchestratorContext = new DSSOrchestratorContextImpl(); + orchestratorContext.initialize(); + } + } + } + } + + private String name; + + private List linkedAppConn = new ArrayList<>(); + + private List labels = new ArrayList<>(); + + private AppConn appConn; + + private SchedulerAppConn schedulerAppConn; + + public DefaultOrchestrator(String name) { + this.name = name; + } + + public void setAppConn(AppConn appConn){ + this.appConn = appConn; + } + + @Override + public String getName() { + return name; + } + + @Override + public AppConn getAppConn() { + return this.appConn; + } + + @Override + public SchedulerAppConn getSchedulerAppConn() { + return schedulerAppConn; + } + + public void setSchedulerAppConn(SchedulerAppConn schedulerAppConn) { + this.schedulerAppConn = schedulerAppConn; + } + + /** + * 添加当前编排需要使用到的 AppConn + * @param appConn 添加当前编排需要使用到的 AppConn + */ + public void addLinkedAppConn(AppConn appConn) { + linkedAppConn.add(appConn); + } + + /** + * 为编排提供标签说明,如DEV + * @param dssLabel 标签 + */ + public void addLinkedDssLabels(DSSLabel dssLabel) { + labels.add(dssLabel); + } + + @Override + public List getLinkedAppConn() { + return linkedAppConn; + } + + @Override + protected DSSOrchestratorContext createOrchestratorContext() { + initDSSOrchestratorContext(); + return orchestratorContext; + } +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/plugin/AbstractDSSOrchestratorPlugin.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/plugin/AbstractDSSOrchestratorPlugin.java new file mode 100644 index 000000000..7bff43478 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/plugin/AbstractDSSOrchestratorPlugin.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.plugin; + +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class AbstractDSSOrchestratorPlugin implements DSSOrchestratorPlugin { + + protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); + + @Override + public void init() { + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void close() throws IOException { + } +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/plugin/DSSOrchestratorPlugin.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/plugin/DSSOrchestratorPlugin.java new file mode 100644 index 000000000..fde7a4117 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/plugin/DSSOrchestratorPlugin.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.plugin; + +import java.io.Closeable; + + +public interface DSSOrchestratorPlugin extends Closeable { + + void init(); + + boolean isReady(); + +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/DSSOrchestratorRelation.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/DSSOrchestratorRelation.java new file mode 100644 index 000000000..af01ab93d --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/DSSOrchestratorRelation.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.orchestrator.core.type; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; + +import java.util.function.Predicate; + +/** + * @author enjoyyin + * @date 2022-03-20 + * @since 1.1.0 + */ +public interface DSSOrchestratorRelation { + + String getDSSOrchestratorMode(); + + String getDSSOrchestratorName(); + + String getDSSOrchestratorName_CN(); + + /** + * 绑定的 AppConn + * @return 绑定的 AppConn + */ + String getBindingAppConnName(); + + String getBindingSchedulerAppConnName(); + + Predicate isLinkedAppConn(); + +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/DSSOrchestratorRelationManager.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/DSSOrchestratorRelationManager.java new file mode 100644 index 000000000..32e796e54 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/DSSOrchestratorRelationManager.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.dss.orchestrator.core.type; + +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.BDPConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * @author enjoyyin + * @date 2022-03-20 + * @since 1.1.0 + */ +public class DSSOrchestratorRelationManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSOrchestratorRelationManager.class); + + private static final List dssOrchestratorRelations = ClassUtils.getInstances(DSSOrchestratorRelation.class, c -> c != OrchestratorJsonRelation.class); + + static { + int index = 1; + while(true) { + String key = "wds.dss.orchestrator.new." + index + "th"; + if(BDPConfiguration.contains(key)) { + String relation = BDPConfiguration.get(key); + OrchestratorJsonRelation jsonRelation = new OrchestratorJsonRelation(relation); + jsonRelation.init(); + LOGGER.info("loaded DSSOrchestratorRelation is {}.", relation); + dssOrchestratorRelations.add(jsonRelation); + } else { + break; + } + index ++; + } + BiConsumer, String> getRepeatRelations = (getField, fieldName) -> { + String repeatRelations = dssOrchestratorRelations.stream().map(getField).collect(Collectors.groupingBy(Function.identity(), Collectors.counting())) + .entrySet().stream().filter(entry -> entry.getValue() > 1).map(Map.Entry::getKey) + .collect(Collectors.joining(", ")); + if(StringUtils.isNotEmpty(repeatRelations)) { + throw new ExternalOperationWarnException(50035, "repeated DSSOrchestratorRelation in " + fieldName + ": " + repeatRelations); + } + }; + getRepeatRelations.accept(DSSOrchestratorRelation::getDSSOrchestratorName, "name"); + getRepeatRelations.accept(DSSOrchestratorRelation::getDSSOrchestratorMode, "mode"); + LOGGER.info("The DSSOrchestratorRelation list is {}.", dssOrchestratorRelations.stream().map(DSSOrchestratorRelation::getDSSOrchestratorName) + .collect(Collectors.joining(", "))); + } + + public static List getDSSOrchestratorRelations() { + return ListUtils.unmodifiableList(dssOrchestratorRelations); + } + + public static DSSOrchestratorRelation getDSSOrchestratorRelationByName(String dssOrchestratorName) { + return dssOrchestratorRelations.stream().filter(dssOrchestratorRelation -> dssOrchestratorRelation.getDSSOrchestratorName().equals(dssOrchestratorName)) + .findAny().orElseThrow(() -> new ExternalOperationWarnException(50355, "Not exists DSSOrchestrator name " + dssOrchestratorName)); + } + + public static DSSOrchestratorRelation getDSSOrchestratorRelationByMode(String dssOrchestratorMode) { + return dssOrchestratorRelations.stream().filter(dssOrchestratorRelation -> dssOrchestratorRelation.getDSSOrchestratorMode().equals(dssOrchestratorMode)) + .findAny().orElseThrow(() -> new ExternalOperationWarnException(50355, "Not exists DSSOrchestrator mode " + dssOrchestratorMode)); + } + +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/OrchestratorJsonRelation.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/OrchestratorJsonRelation.java new file mode 100644 index 000000000..33b3b157e --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/OrchestratorJsonRelation.java @@ -0,0 +1,92 @@ +package com.webank.wedatasphere.dss.orchestrator.core.type; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; + +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Predicate; + +/** + * @author enjoyyin + * @date 2022-03-20 + * @since 0.5.0 + */ +public class OrchestratorJsonRelation implements DSSOrchestratorRelation { + + private String relationString; + private Map relationMap; + private String mode; + private String name; + private String nameCN; + private String schedulerAppConnName; + private String bindingAppConnName; + private Predicate predicate; + + public OrchestratorJsonRelation(String relationString) { + this.relationString = relationString; + } + + public void init() { + relationMap = DSSCommonUtils.COMMON_GSON.fromJson(relationString, Map.class); + Consumer checkConsumer = key -> { + if(!relationMap.containsKey(key)) { + throw new ExternalOperationWarnException(50035, "the relation does not contains ['" + key + "'], relation string is " + relationString); + } + }; + checkConsumer.accept("mode"); + checkConsumer.accept("name"); + checkConsumer.accept("name_cn"); + checkConsumer.accept("bindingAppConnName"); + mode = (String) relationMap.get("mode"); + name = (String) relationMap.get("name"); + nameCN = (String) relationMap.get("nameCN"); + bindingAppConnName = (String) relationMap.get("bindingAppConnName"); + schedulerAppConnName = (String) relationMap.get("schedulerAppConnName"); + if(StringUtils.isBlank(schedulerAppConnName)) { + schedulerAppConnName = null; + } + String linkedAppConn = (String) relationMap.get("linkedAppConn"); + if(StringUtils.isNotBlank(linkedAppConn)) { + String[] appConns = linkedAppConn.split(","); + predicate = appConn -> ArrayUtils.contains(appConns, appConn.getAppDesc().getAppName()); + } else { + predicate = appConn -> appConn instanceof OnlyDevelopmentAppConn; + } + } + + @Override + public String getDSSOrchestratorMode() { + return mode; + } + + @Override + public String getDSSOrchestratorName() { + return name; + } + + @Override + public String getDSSOrchestratorName_CN() { + return nameCN; + } + + @Override + public String getBindingAppConnName() { + return bindingAppConnName; + } + + @Override + public String getBindingSchedulerAppConnName() { + return schedulerAppConnName; + } + + @Override + public Predicate isLinkedAppConn() { + return predicate; + } + +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/WorkflowDSRelation.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/WorkflowDSRelation.java new file mode 100644 index 000000000..b5acb5ff1 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/WorkflowDSRelation.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.dss.orchestrator.core.type; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; + +import java.util.function.Predicate; + +public class WorkflowDSRelation implements DSSOrchestratorRelation { + + @Override + public String getDSSOrchestratorMode() { + return "pom_work_flow_ds"; + } + + @Override + public String getDSSOrchestratorName() { + return "workflow_ds"; + } + + @Override + public String getDSSOrchestratorName_CN() { + return "DS工作流"; + } + + @Override + public String getBindingAppConnName() { + return "workflow"; + } + + @Override + public String getBindingSchedulerAppConnName() { + return "dolphinscheduler"; + } + + @Override + public Predicate isLinkedAppConn() { + return appConn -> appConn instanceof OnlyDevelopmentAppConn; + } +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/WorkflowRelation.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/WorkflowRelation.java new file mode 100644 index 000000000..70d35d670 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/type/WorkflowRelation.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.orchestrator.core.type; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; + +import java.util.function.Predicate; + +/** + * @author enjoyyin + * @date 2022-03-20 + * @since 0.5.0 + */ +public class WorkflowRelation implements DSSOrchestratorRelation { + + @Override + public String getDSSOrchestratorMode() { + return "pom_work_flow"; + } + + @Override + public String getDSSOrchestratorName() { + return "workflow"; + } + + @Override + public String getDSSOrchestratorName_CN() { + return "工作流"; + } + + @Override + public String getBindingAppConnName() { + return "workflow"; + } + + @Override + public String getBindingSchedulerAppConnName() { + return "schedulis"; + } + + @Override + public Predicate isLinkedAppConn() { + return appConn -> appConn instanceof OnlyDevelopmentAppConn; + } +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/utils/OrchestratorUtils.java b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/utils/OrchestratorUtils.java new file mode 100644 index 000000000..c239e5236 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/java/com/webank/wedatasphere/dss/orchestrator/core/utils/OrchestratorUtils.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.utils; + + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class OrchestratorUtils { + + public static final String MODE_SPLIT = ","; + + public static String generateNewVersion() { + return "v000001"; + } + + /** + * 注意: flow版本更新需要同步更新ContextID + * + * @param oldVersion + * @return + */ + public static String increaseVersion(String oldVersion) { + int num = Integer.parseInt(oldVersion.substring(1)) + 1; + String tmp = "00000" + num; + return "v" + tmp.substring(tmp.length() - 6); + } + + //拼接 前后使用英文逗号结尾 + public static String getModeStr(List strList) { + if (CollectionUtils.isEmpty(strList)) { + return null; + } + return strList.stream().map(String::trim).filter((s) -> StringUtils.isNotBlank(s)).distinct().collect(Collectors.joining(MODE_SPLIT, MODE_SPLIT, MODE_SPLIT)); + } + + public static List convertList(String str){ + if(StringUtils.isEmpty(str)){ + return new ArrayList<>(); + } + return Arrays.stream(str.split(MODE_SPLIT)).map(String::trim).filter((s)-> StringUtils.isNotBlank(s)).distinct().collect(Collectors.toList()); + } +} diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/scala/com/webank/wedatasphere/dss/orchestrator/core/service/BMLService.scala b/dss-orchestrator/dss-orchestrator-core/src/main/scala/com/webank/wedatasphere/dss/orchestrator/core/service/BMLService.scala new file mode 100644 index 000000000..133115a18 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/scala/com/webank/wedatasphere/dss/orchestrator/core/service/BMLService.scala @@ -0,0 +1,183 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.core.service + +import java.io.{ByteArrayInputStream, InputStream} +import java.util +import java.util.UUID + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException +import com.webank.wedatasphere.dss.common.utils.IoUtils +import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory} +import org.apache.linkis.bml.protocol.{BmlDownloadResponse, BmlUpdateResponse, BmlUploadResponse} +import org.apache.linkis.common.utils.{JavaLog, Utils} +import org.apache.commons.io.IOUtils +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +@Component +class BMLService extends JavaLog{ + + def upload(userName: String, content: String, fileName: String, projectName:String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = createBMLClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def upload(userName: String, inputStream: InputStream, fileName: String, projectName:String): util.Map[String, Object] = { + val client: BmlClient = createBMLClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def update(userName: String, resourceId: String, inputStream: InputStream): util.Map[String, Object] = { + val client: BmlClient = createBMLClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, "", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def update(userName: String, resourceId: String, content: String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = createBMLClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, UUID.randomUUID().toString+".json", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def query(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = createBMLClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "string" -> inputstremToString(resource.inputStream) + } + + def download(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = createBMLClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "is" -> resource.inputStream + } + + def downloadToLocalPath(userName: String, resourceId: String, version: String, path: String): String = { + val result = download(userName,resourceId,version) + val is = result.get("is").asInstanceOf[InputStream] + val os = IoUtils.generateExportOutputStream(path) + Utils.tryFinally(IOUtils.copy(is,os)){ + IOUtils.closeQuietly(os) + IOUtils.closeQuietly(is) + } + path + } + + def downloadAndGetFlowJson(userName: String, resourceId: String, version: String, path: String): String = { + downloadToLocalPath(userName,resourceId,version,path) + //因为下载到指定目录后返回的resource的Stream为null,只能从文件重新读取。 + val is = IoUtils.generateInputInputStream(path) + Utils.tryFinally(inputstremToString(is))(IOUtils.closeQuietly(is)) + } + + def readLocalResourceFile(userName: String,readPath: String): InputStream ={ + IoUtils.generateInputInputStream(readPath) + } + + def readLocalFlowJsonFile(userName: String,readPath: String): String ={ + var inputStream:InputStream = null + var flowJson:String = null + Utils.tryFinally{ + inputStream = IoUtils.generateInputInputStream(readPath) + flowJson = inputstremToString(inputStream) + }{ + IOUtils.closeQuietly(inputStream) + } + flowJson + } + + private def inputstremToString(inputStream: InputStream): String = { + scala.io.Source.fromInputStream(inputStream).mkString + } + + private def createBMLClient(userName: String): BmlClient = { + if (userName == null) + BmlClientFactory.createBmlClient() + else + BmlClientFactory.createBmlClient(userName) + } + + def createBmlProject(username:String, projectName:String, editUsers:util.List[String], + accessUsers:util.List[String] ): Unit ={ + val client = createBMLClient(username) + val response = client.createBmlProject(username, projectName, accessUsers, editUsers) + if (response.isSuccess){ + logger.info(s"for user $username create bml project $projectName success") + }else{ + logger.error(s"for user $username create bml project $projectName failed") + } + } + + def attachResourceAndProject(username:String, projectName:String, resourceId:String):Unit = { + val client = createBMLClient(username) + val response = client.attachResourceAndProject(projectName, resourceId) + if (response.isSuccess){ + logger.info(s"attach $username and $projectName success") + }else{ + logger.error(s"attach $username and $projectName failed") + } + } + + + def updateProjectPriv(projectName:String, username:String, editUsers:util.List[String], + accessUsers:util.List[String]): Unit ={ + val client = createBMLClient(username) + val response = client.updateProjectPriv(username, projectName, editUsers, accessUsers) + if (response.isSuccess){ + logger.info(s"attach $username and $projectName success") + }else{ + logger.error(s"attach $username and $projectName failed") + } + } + + + +} diff --git a/dss-orchestrator/dss-orchestrator-db/pom.xml b/dss-orchestrator/dss-orchestrator-db/pom.xml new file mode 100644 index 000000000..f0d23a2fa --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-db/pom.xml @@ -0,0 +1,77 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-orchestrator-db + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + provided + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.xml + **/*.properties + **/*.yml + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorJobMapper.java b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorJobMapper.java new file mode 100644 index 000000000..306c253be --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorJobMapper.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.orchestrator.db.dao; + +import com.webank.wedatasphere.dss.orchestrator.db.entity.OrchestratorPublishJob; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface OrchestratorJobMapper { + + OrchestratorPublishJob getPublishJobById(long id); + + OrchestratorPublishJob getPublishJobByJobId(String jobId); + + long insertPublishJob(OrchestratorPublishJob job); + + void updatePublishJob(OrchestratorPublishJob job); + + void deletePublishJob(OrchestratorPublishJob job); + +} diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java new file mode 100644 index 000000000..b2490d1b7 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java @@ -0,0 +1,126 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.db.dao; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; +import org.apache.ibatis.annotations.*; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Mapper +public interface OrchestratorMapper { + + void addOrchestrator(DSSOrchestratorInfo dssOrchestratorInfo); + + DSSOrchestratorInfo getOrchestrator(Long id); + + DSSOrchestratorInfo getOrchestratorByUUID(String uuid); + + void updateOrchestrator(DSSOrchestratorInfo dssOrchestratorInfo); + + void deleteOrchestrator(Long id); + + /** + * @param dssOrchestratorVersion + */ + + void addOrchestratorVersion(DSSOrchestratorVersion dssOrchestratorVersion); + + DSSOrchestratorVersion getOrchestratorVersion(Long versionId); + + /** + * 获取orc指定版本信息 + * + * @param orchestratorId id + * @param orcVersionId orc版本id + * @param validFlag 有效标志 + * @return DSSOrchestratorVersion + */ + DSSOrchestratorVersion getOrcVersionByIdAndOrcVersionId(@Param("orchestratorId") Long orchestratorId, + @Param("orcVersionId") Long orcVersionId, + @Param("validFlag") Integer validFlag); + + + /** + * 根据id查找最新的版本信息 + * + * @param orchestratorId + * @return + */ + DSSOrchestratorVersion getLatestOrchestratorVersionById(Long orchestratorId); + + /** + * 根据id查找最新的版本信息 + * + * @param orchestratorId + * @return + */ + DSSOrchestratorVersion getLatestOrchestratorVersionByIdAndValidFlag(@Param("orchestratorId") Long orchestratorId, @Param("validFlag") Integer validFlag); + + void updateOrchestratorVersion(DSSOrchestratorVersion dssOrchestratorVersion); + + void deleteOrchestratorVersion(Long versionId); + + /** + * 根据id查找所有的版本信息 + * + * @param orchestratorId + * @return + */ + List getVersionByOrchestratorId(Long orchestratorId); + + OrchestratorInfo getOrcInfoByAppId(@Param("appId") Long appId); + + @Select("select max(id) from dss_orchestrator_version_info where `orchestrator_id` = #{orchestratorId}") + Long findLatestOrcVersionId(@Param("orchestratorId") Long orchestratorId); + + @Select("select id from dss_orchestrator_version_info where `orchestrator_id` = #{orchestratorId} and valid_flag = #{validFlag} ORDER BY version DESC LIMIT 1") + Long findLatestVIdByOrcIdAndValidFlag(@Param("orchestratorId") Long orchestratorId, @Param("validFlag") Integer validFlag); + + List getOrchestratorVersions(@Param("projectId") Long projectId, @Param("orchestratorId") Long orchestratorId); + + @Select("select max(`version`) from dss_orchestrator_version_info where orchestrator_id = #{orchestratorId} and valid_flag = #{validFlag}") + String getLatestVersion(@Param("orchestratorId") Long orchestratorId, @Param("validFlag") Integer validFlag); + + @Select("select uuid from dss_orchestrator_info where project_id = #{projectId} and name = #{name}") + String getOrcNameByParam(@Param("projectId") Long projectId, @Param("name") String name); + + @Select("select id from `dss_orchestrator_info` where `project_id` = #{projectId} and `is_published` = 1") + List getAllOrcIdsByProjectId(@Param("projectId") Long projectId); + + @Select("select max(`app_id`) from `dss_orchestrator_version_info` where `orchestrator_id` = #{orchestratorId} and `version` = #{version}") + Long getAppIdByVersion(@Param("orchestratorId") Long orchestratorId, @Param("version") String version); + + DSSOrchestratorVersion getVersionByOrchestratorIdAndVersion(@Param("orchestratorId") Long orchestratorId, @Param("version") String version); + + @Update("update `dss_orchestrator_info` set `is_published` = 1 where id = #{orchestratorId}") + void setPublished(@Param("orchestratorId") Long orchestratorId); + + @Select("select `name` from dss_orchestrator_info where `id` = #{orchestratorId}") + String getOrchestratorNameById(@Param("orchestratorId") int orchestratorId); + + @Select("select * from dss_orchestrator_info where `project_id` = #{projectId} and `name` = #{name}") + List getByNameAndProjectId(@Param("projectId") Long projectId, @Param("name") String name); + + void addOrchestratorRefOrchestration(DSSOrchestratorRefOrchestration dssOrchestratorRefOrchestration); + + List queryOrchestratorInfos(@Param("params") Map params); + + DSSOrchestratorRefOrchestration getRefOrchestrationId(@Param("orchestratorId") Long orchestratorId); +} diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/OrchestratorJobMapper.xml b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/OrchestratorJobMapper.xml new file mode 100644 index 000000000..7fcf5d70a --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/OrchestratorJobMapper.xml @@ -0,0 +1,69 @@ + + + + + + + + + + id, job_id, conversion_job_json, created_time, updated_time + + + + + + + + INSERT INTO dss_orchestrator_job_info () + VALUES + (#{id},#{jobId},#{conversionJobJson},#{createdTime}) + + + + UPDATE dss_orchestrator_job_info + + job_id =#{jobId}, + conversion_job_json =#{conversionJobJson}, + created_time =#{createdTime}, + updated_time =#{updatedTime}, + + WHERE id = #{id} + + + + DELETE + FROM + dss_orchestrator_job_info + WHERE id = #{id} + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml new file mode 100644 index 000000000..edc7f455e --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml @@ -0,0 +1,242 @@ + + + + + + + + + + id,`name`,`type`,`desc`,`creator`,`create_time`,`project_id`,`uses`,`appconn_name`,`uuid`,`secondary_type` , + `workspace_id`,`orchestrator_mode`,`orchestrator_way`,`orchestrator_level`,`update_user`,`update_time` + + + + id,`orchestrator_id`,`app_id`,`source`,`version`,`comment`,`update_time`,`updater`,`project_id`,`content`,`context_id`,`valid_flag` + + + + id,`orchestrator_id`,`ref_project_id`,`ref_orchestration_id` + + + + + + + + INSERT INTO dss_orchestrator_info () + VALUES + (#{id},#{name},#{type},#{desc},#{creator},#{createTime},#{projectId},#{uses},#{appConnName}, + #{uuid},#{secondaryType}, + #{workspaceId},#{orchestratorMode},#{orchestratorWay},#{orchestratorLevel},#{updateUser},#{updateTime}) + + + + UPDATE dss_orchestrator_info + + `name`=#{name}, + `desc`=#{desc}, + `appconn_name`=#{appConnName}, + `uses`=#{uses}, + `secondary_type`=#{secondaryType}, + `orchestrator_level`=#{orchestratorLevel}, + `update_user`=#{updateUser}, + `update_time`=#{updateTime}, + + WHERE id = #{id} + + + + + DELETE + FROM + dss_orchestrator_info + WHERE id = #{id} + + + + INSERT INTO dss_orchestrator_version_info () + VALUES + (#{id},#{orchestratorId},#{appId},#{source},#{version},#{comment},#{updateTime},#{updater},#{projectId},#{content},#{contextId},#{validFlag}) + + + + + UPDATE dss_orchestrator_version_info + + `source` = #{source}, + `valid_flag` = #{validFlag}, + `app_id` = #{appId}, + `content`= #{content}, + `comment` = #{comment}, + `context_id` = #{contextId}, + `update_time` = #{updateTime}, + `updater` = #{updater} + + WHERE id = #{id} + + + + + + + + + + + + + DELETE + FROM + dss_orchestrator_version_info + WHERE id = #{versionId} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + insert into dss_orchestrator_ref_orchestration_relation () + values (#{id},#{orchestratorId},#{refProjectId},#{refOrchestrationId}) + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/entity/OrchestratorPublishJob.java b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/entity/OrchestratorPublishJob.java new file mode 100644 index 000000000..854eebbfa --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/entity/OrchestratorPublishJob.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.dss.orchestrator.db.entity; + +import java.util.Date; + +public class OrchestratorPublishJob { + + private Long id; + private String jobId; + private String conversionJobJson; + private Date createdTime; + private Date updatedTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getJobId() { + return jobId; + } + + public OrchestratorPublishJob setJobId(String jobId) { + this.jobId = jobId; + return this; + } + + public String getConversionJobJson() { + return conversionJobJson; + } + + public OrchestratorPublishJob setConversionJobJson(String conversionJobJson) { + this.conversionJobJson = conversionJobJson; + return this; + } + + public Date getCreatedTime() { + return createdTime; + } + + public OrchestratorPublishJob setCreatedTime(Date createdTime) { + this.createdTime = createdTime; + return this; + } + + public Date getUpdatedTime() { + return updatedTime; + } + + public OrchestratorPublishJob setUpdatedTime(Date updatedTime) { + this.updatedTime = updatedTime; + return this; + } +} diff --git a/dss-orchestrator/dss-orchestrator-loader/pom.xml b/dss-orchestrator/dss-orchestrator-loader/pom.xml new file mode 100644 index 000000000..3082f7c46 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-loader/pom.xml @@ -0,0 +1,76 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-orchestrator-loader + + + jakarta.annotation + jakarta.annotation-api + 1.3.5 + provided + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + + + com.webank.wedatasphere.dss + dss-orchestrator-db + ${dss.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.springframework + spring-context + ${spring.version} + provided + + + com.webank.wedatasphere.dss + dss-appconn-manager-core + ${dss.version} + + + org.apache.commons + commons-math3 + provided + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/DefaultLinkedAppConnResolver.java b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/DefaultLinkedAppConnResolver.java new file mode 100644 index 000000000..9db2b1255 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/DefaultLinkedAppConnResolver.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.loader; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + + + +@Component +class DefaultLinkedAppConnResolver implements LinkedAppConnResolver { + + @Override + public List resolveAppConnByUser(String userName, String workspaceName, String typeName) { + //todo 后面可以使用数据库表来定义用户可以加载的AppConn. + List appConns = new ArrayList<>(); + for(AppConn appConn : AppConnManager.getAppConnManager().listAppConns()){ + //可以在这里根据用户情况和工作空间情况,限制appConn的加载 + appConns.add(appConn); + } + + return appConns; + } +} diff --git a/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/DefaultOrchestratorLoader.java b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/DefaultOrchestratorLoader.java new file mode 100644 index 000000000..6255c680a --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/DefaultOrchestratorLoader.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.loader; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext; +import com.webank.wedatasphere.dss.orchestrator.core.impl.DefaultOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.core.type.DSSOrchestratorRelation; +import com.webank.wedatasphere.dss.orchestrator.core.type.DSSOrchestratorRelationManager; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; + + +@Component +public class DefaultOrchestratorLoader implements OrchestratorLoader { + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultOrchestratorLoader.class); + @Autowired + private DSSOrchestratorContext dssOrchestratorContext; + + @Autowired + private LinkedAppConnResolver linkedAppConnResolver; + + + @Override + public DSSOrchestrator loadOrchestrator(String userName, + String workspaceName, + String typeName, + List dssLabels) { + + DefaultOrchestrator dssOrchestrator = new DefaultOrchestrator(typeName) { + @Override + protected DSSOrchestratorContext createOrchestratorContext() { + return dssOrchestratorContext; + } + }; + DSSOrchestratorRelation relation = DSSOrchestratorRelationManager.getDSSOrchestratorRelationByName(typeName); + // 添加实现了第三级规范的AppConn + List appConnList = linkedAppConnResolver.resolveAppConnByUser(userName, workspaceName, typeName); + appConnList.stream().filter(relation.isLinkedAppConn()).forEach(dssOrchestrator::addLinkedAppConn); + AppConn appConn = AppConnManager.getAppConnManager().getAppConn(relation.getBindingAppConnName()); + dssOrchestrator.setAppConn(appConn); + if (appConnList.stream().anyMatch(t -> t instanceof SchedulerAppConn)) { + List schedulerAppConns = AppConnManager.getAppConnManager().listAppConns(SchedulerAppConn.class); + SchedulerAppConn schedulerAppConn; + if(StringUtils.isBlank(relation.getBindingSchedulerAppConnName())) { + schedulerAppConn = schedulerAppConns.get(0); + } else { + schedulerAppConn = schedulerAppConns.stream().filter(appConn1 -> appConn1.getAppDesc().getAppName().equals(relation.getBindingSchedulerAppConnName())) + .findAny().orElseThrow(() -> new ExternalOperationFailedException(50032, "cannot find the matched SchedulerAppConn with name " + relation.getBindingSchedulerAppConnName())); + } + dssOrchestrator.setSchedulerAppConn(schedulerAppConn); + LOGGER.info("Load dss orchestrator: {}, with the binding AppConn: {} and binding SchedulerAppConn {}.", + typeName, relation.getBindingAppConnName(), schedulerAppConn.getAppDesc().getAppName()); + } + dssLabels.forEach(dssOrchestrator::addLinkedDssLabels); + return dssOrchestrator; + } +} diff --git a/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/LinkedAppConnResolver.java b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/LinkedAppConnResolver.java new file mode 100644 index 000000000..66bc4c748 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/LinkedAppConnResolver.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.loader; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; + + +import java.util.List; + +public interface LinkedAppConnResolver { + List resolveAppConnByUser(String userName, String workspaceName, String typeName); +} diff --git a/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/OrchestratorLoader.java b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/OrchestratorLoader.java new file mode 100644 index 000000000..0e326463f --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/OrchestratorLoader.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.loader; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; + +import java.util.List; + +public interface OrchestratorLoader { + + /** + * 用于返回一个指定类型的的Orchestrator + * @param userName + * @param workspaceName + * @param typeName + * @return + */ + DSSOrchestrator loadOrchestrator(String userName, + String workspaceName, + String typeName, + List dssLabels); +} diff --git a/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/OrchestratorManager.java b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/OrchestratorManager.java new file mode 100644 index 000000000..7a301cf0c --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/OrchestratorManager.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.loader; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + + +@Component +public class OrchestratorManager { + + private final Map cacheDssOrchestrator = new ConcurrentHashMap<>(); + + @Autowired + private DefaultOrchestratorLoader defaultOrchestratorLoader; + + public DSSOrchestrator getOrCreateOrchestrator(String userName, + String workspaceName, + String typeName, + List dssLabels) { + String findKey = getCacheKey(userName, workspaceName, typeName); + DSSOrchestrator dssOrchestrator = cacheDssOrchestrator.get(findKey); + if (null == dssOrchestrator) { + synchronized (cacheDssOrchestrator) { + dssOrchestrator = cacheDssOrchestrator.get(findKey); + if(null == dssOrchestrator) { + dssOrchestrator = defaultOrchestratorLoader.loadOrchestrator(userName, workspaceName, typeName, dssLabels); + cacheDssOrchestrator.put(findKey, dssOrchestrator); + } + } + } + return dssOrchestrator; + } + + protected String getCacheKey(String userName, String workspaceName, String typeName) { + return userName + "_" + workspaceName + "_" + typeName; + } +} diff --git a/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/SpringDSSOrchestratorContext.java b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/SpringDSSOrchestratorContext.java new file mode 100644 index 000000000..d1591764c --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/SpringDSSOrchestratorContext.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.loader; + +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.orchestrator.core.impl.AbstractDSSOrchestratorContext; +import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + + +@Component +public class SpringDSSOrchestratorContext extends AbstractDSSOrchestratorContext { + + private final Logger LOGGER = LoggerFactory.getLogger(SpringDSSOrchestratorContext.class); + + @Autowired + private List dssOrchestratorPlugins; + + @Override + @PostConstruct + public void initialize() { + if(dssOrchestratorPlugins == null || dssOrchestratorPlugins.isEmpty()) { + dssOrchestratorPlugins = ClassUtils.getInstances(DSSOrchestratorPlugin.class); + } else { + List others = ClassUtils.getClasses(DSSOrchestratorPlugin.class).stream() + .filter(clazz -> dssOrchestratorPlugins.stream().noneMatch(clazz::isInstance)) + .map(DSSExceptionUtils.map(Class::newInstance)).collect(Collectors.toList()); + dssOrchestratorPlugins.addAll(others); + } + dssOrchestratorPlugins.forEach(DSSOrchestratorPlugin::init); + LOGGER.info("The DSSOrchestratorPlugin list is {}.", dssOrchestratorPlugins); + } + + @Override + public List getOrchestratorPlugins() { + return dssOrchestratorPlugins; + } + + @Override + @PreDestroy + public void close() { + super.close(); + } +} diff --git a/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/utils/OrchestratorLoaderUtils.java b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/utils/OrchestratorLoaderUtils.java new file mode 100644 index 000000000..34b67831a --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/utils/OrchestratorLoaderUtils.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.orchestrator.loader.utils; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.loader.OrchestratorManager; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.exception.NoSuchAppInstanceException; +import org.apache.linkis.protocol.util.ImmutablePair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Component +public class OrchestratorLoaderUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(OrchestratorLoaderUtils.class); + @Autowired + private OrchestratorManager orchestratorManager; + private static OrchestratorLoaderUtils orchestratorLoaderUtils; + + @PostConstruct + public void init(){ + orchestratorLoaderUtils = this; + } + + public static ImmutablePair getOrchestratorDevelopmentStandard( + DSSOrchestrator dssOrchestrator, List dssLabels) throws NoSuchAppInstanceException { + AppConn orchestratorAppConn = dssOrchestrator.getAppConn(); + if(orchestratorAppConn instanceof OnlyDevelopmentAppConn) { + DevelopmentIntegrationStandard developmentIntegrationStandard = ((OnlyDevelopmentAppConn) orchestratorAppConn).getOrCreateDevelopmentStandard(); + List appInstance = orchestratorAppConn.getAppDesc().getAppInstancesByLabels(dssLabels); + if (appInstance.size() > 0 && null != developmentIntegrationStandard) { + return new ImmutablePair<>(appInstance.get(0), developmentIntegrationStandard); + } else { + throw new NoSuchAppInstanceException(50635, "The binding AppConn of Orchestrator " + dssOrchestrator.getName() + + " has no appInstance."); + } + } else { + throw new NoSuchAppInstanceException(50635, "The binding AppConn of Orchestrator " + dssOrchestrator.getName() + + " has not implements OnlyDevelopmentAppConn."); + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml new file mode 100644 index 000000000..0ae5a3f9e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml @@ -0,0 +1,229 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-flow-execution-server + + + + org.apache.linkis + linkis-entrance + ${linkis.version} + + + linkis-common + org.apache.linkis + + + spring-tx + org.springframework + + + commons-text + org.apache.commons + + + json4s-jackson_2.11 + org.json4s + + + linkis-label-common + org.apache.linkis + + + + + + com.webank.wedatasphere.dss + dss-linkis-node-execution + ${dss.version} + + + linkis-common + org.apache.linkis + + + commons-beanutils + commons-beanutils + + + linkis-label-common + org.apache.linkis + + + + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + + + javax.validation + validation-api + + + org.glassfish.hk2.external + bean-validator + + + + + + org.apache.linkis + linkis-storage + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-workflow-sdk + ${dss.version} + + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + + + com.webank.wedatasphere.dss + dss-sender-service + ${dss.version} + provided + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + provided + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + -target:jvm-1.8 + + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + + ${project.artifactId}-${project.version} + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/assembly/distribution.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..d14b0412b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/assembly/distribution.xml @@ -0,0 +1,43 @@ + + + + dss-framework-execution-server + + dir + + true + dss-flow-execution-server + + + + + + lib + true + true + false + true + true + + + + + + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/TaskMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/TaskMapper.java new file mode 100644 index 000000000..af84a34ce --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/TaskMapper.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.dao; + +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.WorkflowQueryTask; +import org.apache.ibatis.annotations.Param; +import java.util.Date; +import java.util.List; + + +public interface TaskMapper { + + List selectTask(WorkflowQueryTask queryTask); + + void insertTask(WorkflowQueryTask queryTask); + + void updateTask(WorkflowQueryTask queryTask); + + List search(@Param("taskID") Long taskID, @Param("umUser") String username, @Param("status") List status, + @Param("startDate") Date startDate, @Param("endDate") Date endDate, @Param("executeApplicationName") String executeApplicationName, + @Param("instance") String instance, @Param("execId") String execId); + + String selectTaskStatusForUpdate(Long taskID); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/WorkflowExecuteInfoMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/WorkflowExecuteInfoMapper.java new file mode 100644 index 000000000..a078c7b58 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/WorkflowExecuteInfoMapper.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.dao; + +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.WorkflowExecuteInfo; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +public interface WorkflowExecuteInfoMapper { + + int insert(WorkflowExecuteInfo workflowExecuteInfo); + + int updateWorkflowExecuteInfo(WorkflowExecuteInfo workflowExecuteInfo); + + WorkflowExecuteInfo getExecuteInfoByFlowId(@Param("flowId") Long flowId); + + WorkflowExecuteInfo getExecuteInfoByFlowIdAndVersion(@Param("flowId") Long flowId,@Param("version") String version); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/WorkflowExecuteInfoMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/WorkflowExecuteInfoMapper.xml new file mode 100644 index 000000000..8f326cb00 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/WorkflowExecuteInfoMapper.xml @@ -0,0 +1,50 @@ + + + + + + + + + + task_id,flow_id,version,status,failed_jobs,Pending_jobs,skipped_jobs,succeed_jobs,running_jobs,createtime + + + + INSERT INTO dss_workflow_execute_info() + VALUES + (#{taskId},#{flowId},#{version},#{status},#{failedJobs},#{PendingJobs},#{skippedJobs},#{succeedJobs},#{runningJobs},#{createtime}) + + + + + + + + update dss_workflow_execute_info set task_id = #{taskId},status = #{status}, + failed_jobs = #{failedJobs},Pending_jobs = #{PendingJobs},skipped_jobs = #{skippedJobs}, + succeed_jobs = #{succeedJobs},running_jobs = #{runningJobs},updatetime = now() where id = #{id} + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/taskMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/taskMapper.xml new file mode 100644 index 000000000..d43e9763f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/taskMapper.xml @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + id, `instance`, `exec_id`, `um_user`, `execution_code`, `progress`,`log_path`, + `result_location`,`status`,`created_time`,`updated_time`,`run_type`,`err_code`,`err_desc`,`execute_application_name`,`request_application_name`,`script_path`,`params`,`engine_instance`, + `engine_start_time`, `task_resource`, `submit_user` , `label_json` + + + + INSERT INTO dss_workflow_task() + VALUES (#{taskID},#{instance},#{execId}, + #{umUser},#{executionCode},#{progress}, + #{logPath},#{resultLocation},#{status}, + #{createdTime},#{updatedTime},#{runType}, + #{errCode},#{errDesc},#{executeApplicationName}, + #{requestApplicationName},#{sourceJson},#{paramsJson}, + #{engineInstance},#{engineStartTime}, #{taskResource}, #{submitUser}, #{labelJson}) + + + + + + + + + UPDATE dss_workflow_task + + instance=#{instance}, + exec_id=#{execId}, + um_user=#{umUser}, + execution_code=#{executionCode}, + progress=#{progress}, + log_path=#{logPath}, + result_location=#{resultLocation}, + status=#{status}, + created_time=#{createdTime}, + updated_time=#{updatedTime}, + run_type=#{runType}, + err_code=#{errCode}, + err_desc=#{errDesc}, + execute_application_name=#{executeApplicationName}, + request_application_name=#{requestApplicationName}, + script_path = #{sourceJson}, + params = #{paramsJson}, + engine_instance = #{engineInstance}, + engine_start_time = #{engineStartTime}, + task_resource = #{taskResource} + + WHERE id =#{taskID} + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowExecuteInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowExecuteInfo.java new file mode 100644 index 000000000..9eafa82e4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowExecuteInfo.java @@ -0,0 +1,117 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.entity; + +import java.util.Date; + +public class WorkflowExecuteInfo { + private Long id; + private Long taskId; + private Long flowId; + private Integer status; + private String version; + private String failedJobs; + private String PendingJobs; + private String skippedJobs; + private String succeedJobs; + private String runningJobs; + private Date createtime; + private Date updatetime; + + public WorkflowExecuteInfo() { + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getTaskId() { + return taskId; + } + + public void setTaskId(Long taskId) { + this.taskId = taskId; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getFailedJobs() { + return failedJobs; + } + + public void setFailedJobs(String failedJobs) { + this.failedJobs = failedJobs; + } + + public String getPendingJobs() { + return PendingJobs; + } + + public void setPendingJobs(String pendingJobs) { + PendingJobs = pendingJobs; + } + + public String getSkippedJobs() { + return skippedJobs; + } + + public void setSkippedJobs(String skippedJobs) { + this.skippedJobs = skippedJobs; + } + + public String getSucceedJobs() { + return succeedJobs; + } + + public void setSucceedJobs(String succeedJobs) { + this.succeedJobs = succeedJobs; + } + + public Date getCreatetime() { + return createtime; + } + + public void setCreatetime(Date createtime) { + this.createtime = createtime; + } + + public String getRunningJobs() { + return runningJobs; + } + + public void setRunningJobs(String runningJobs) { + this.runningJobs = runningJobs; + } + + public Date getUpdatetime() { + return updatetime; + } + + public void setUpdatetime(Date updatetime) { + this.updatetime = updatetime; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowExecuteInfoVo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowExecuteInfoVo.java new file mode 100644 index 000000000..35ee107d2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowExecuteInfoVo.java @@ -0,0 +1,57 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.entity; + +import java.util.List; +import java.util.Map; + +public class WorkflowExecuteInfoVo extends WorkflowExecuteInfo { + List> runningJobsList; + List> failedJobsList; + List> succeedJobsList; + List> pendingJobsList; + List> skippedJobsList; + + public WorkflowExecuteInfoVo() { + } + + public List> getRunningJobsList() { + return runningJobsList; + } + + public void setRunningJobsList(List> runningJobsList) { + this.runningJobsList = runningJobsList; + } + + public List> getFailedJobsList() { + return failedJobsList; + } + + public void setFailedJobsList(List> failedJobsList) { + this.failedJobsList = failedJobsList; + } + + public List> getSucceedJobsList() { + return succeedJobsList; + } + + public void setSucceedJobsList(List> succeedJobsList) { + this.succeedJobsList = succeedJobsList; + } + + public List> getPendingJobsList() { + return pendingJobsList; + } + + public void setPendingJobsList(List> pendingJobsList) { + this.pendingJobsList = pendingJobsList; + } + + public List> getSkippedJobsList() { + return skippedJobsList; + } + + public void setSkippedJobsList(List> skippedJobsList) { + this.skippedJobsList = skippedJobsList; + } + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowQueryTask.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowQueryTask.java new file mode 100644 index 000000000..cbd080812 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowQueryTask.java @@ -0,0 +1,241 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.entity; + +import java.util.Date; + + +public class WorkflowQueryTask { + private Long taskID; + private String instance; + private String execId; + private String umUser; + private String engineInstance; + private String executionCode; + private Float progress; + private String logPath; + private String resultLocation; + private String status; + private Date createdTime; + private Date updatedTime; + private String engineType; + private Integer errCode; + private String errDesc; + private String executeApplicationName; + private String requestApplicationName; + private String runType; + private String paramsJson; + private String sourceJson; + private Date engineStartTime; + private String taskResource; + + private String submitUser; + + private String labelJson; + + public Date getEngineStartTime() { + return engineStartTime; + } + + public void setEngineStartTime(Date engineStartTime) { + this.engineStartTime = engineStartTime; + } + + public String getSourceJson() { + return sourceJson; + } + + public void setSourceJson(String sourceJson) { + this.sourceJson = sourceJson; + } + + public Long getTaskID() { + return taskID; + } + + public void setTaskID(Long taskID) { + this.taskID = taskID; + } + + public String getInstance() { + return instance; + } + + public void setInstance(String instance) { + this.instance = instance; + } + + public String getExecId() { + return execId; + } + + public void setExecId(String execId) { + this.execId = execId; + } + + public String getUmUser() { + return umUser; + } + + public void setUmUser(String umUser) { + this.umUser = umUser; + } + + public String getEngineInstance() { + return engineInstance; + } + + public void setEngineInstance(String engineInstance) { + this.engineInstance = engineInstance; + } + + public String getExecutionCode() { + return executionCode; + } + + public void setExecutionCode(String executionCode) { + this.executionCode = executionCode; + } + + public Float getProgress() { + return progress; + } + + public void setProgress(Float progress) { + this.progress = progress; + } + + public String getLogPath() { + return logPath; + } + + public void setLogPath(String logPath) { + this.logPath = logPath; + } + + public String getResultLocation() { + return resultLocation; + } + + public void setResultLocation(String resultLocation) { + this.resultLocation = resultLocation; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Date getCreatedTime() { + return createdTime; + } + + public void setCreatedTime(Date createdTime) { + this.createdTime = createdTime; + } + + public Date getUpdatedTime() { + return updatedTime; + } + + public void setUpdatedTime(Date updatedTime) { + this.updatedTime = updatedTime; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public Integer getErrCode() { + return errCode; + } + + public void setErrCode(Integer errCode) { + this.errCode = errCode; + } + + public String getErrDesc() { + return errDesc; + } + + public void setErrDesc(String errDesc) { + this.errDesc = errDesc; + } + + public String getExecuteApplicationName() { + return executeApplicationName; + } + + public void setExecuteApplicationName(String executeApplicationName) { + this.executeApplicationName = executeApplicationName; + } + + public String getRequestApplicationName() { + return requestApplicationName; + } + + public void setRequestApplicationName(String requestApplicationName) { + this.requestApplicationName = requestApplicationName; + } + + public String getRunType() { + return runType; + } + + public void setRunType(String runType) { + this.runType = runType; + } + + public String getParamsJson() { + return paramsJson; + } + + public void setParamsJson(String paramsJson) { + this.paramsJson = paramsJson; + } + + public String getTaskResource() { + return taskResource; + } + + public void setTaskResource(String taskResource) { + this.taskResource = taskResource; + } + + public String getSubmitUser() { + return submitUser; + } + + public void setSubmitUser(String submitUser) { + this.submitUser = submitUser; + } + + public String getLabelJson() { + return labelJson; + } + + public void setLabelJson(String labelJson) { + this.labelJson = labelJson; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowQueryTaskVO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowQueryTaskVO.java new file mode 100644 index 000000000..3fc5dfba2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/WorkflowQueryTaskVO.java @@ -0,0 +1,243 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.entity; + +import java.util.Date; + + +public class WorkflowQueryTaskVO { + private Long taskID; + private String instance; + private String execId; + private String umUser; + private String engineInstance; + private String executionCode; + private Float progress; + private String logPath; + private String resultLocation; + private String status; + private Date createdTime; + private Date updatedTime; + private String engineType; + private Integer errCode; + private String errDesc; + private String executeApplicationName; + private String requestApplicationName; + private String runType; + private String paramsJson; + private Long costTime; + private String strongerExecId; + private String sourceJson; + /** + * source字段:用来将sourceJson的value取出来进行拼接返回给前台展示 + */ + private String sourceTailor; + + private Date engineStartTime; + + public Date getEngineStartTime() { + return engineStartTime; + } + + public void setEngineStartTime(Date engineStartTime) { + this.engineStartTime = engineStartTime; + } + + public String getSourceTailor() { + return sourceTailor; + } + + public void setSourceTailor(String sourceTailor) { + this.sourceTailor = sourceTailor; + } + + public String getSourceJson() { + return sourceJson; + } + + public void setSourceJson(String sourceJson) { + this.sourceJson = sourceJson; + } + + public Long getTaskID() { + return taskID; + } + + public void setTaskID(Long taskID) { + this.taskID = taskID; + } + + public String getInstance() { + return instance; + } + + public void setInstance(String instance) { + this.instance = instance; + } + + public String getExecId() { + return execId; + } + + public void setExecId(String execId) { + this.execId = execId; + } + + public String getUmUser() { + return umUser; + } + + public void setUmUser(String umUser) { + this.umUser = umUser; + } + + public String getEngineInstance() { + return engineInstance; + } + + public void setEngineInstance(String engineInstance) { + this.engineInstance = engineInstance; + } + + public String getExecutionCode() { + return executionCode; + } + + public void setExecutionCode(String executionCode) { + this.executionCode = executionCode; + } + + public Float getProgress() { + return progress; + } + + public void setProgress(Float progress) { + this.progress = progress; + } + + public String getLogPath() { + return logPath; + } + + public void setLogPath(String logPath) { + this.logPath = logPath; + } + + public String getResultLocation() { + return resultLocation; + } + + public void setResultLocation(String resultLocation) { + this.resultLocation = resultLocation; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Date getCreatedTime() { + return createdTime; + } + + public void setCreatedTime(Date createdTime) { + this.createdTime = createdTime; + } + + public Date getUpdatedTime() { + return updatedTime; + } + + public void setUpdatedTime(Date updatedTime) { + this.updatedTime = updatedTime; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public Integer getErrCode() { + return errCode; + } + + public void setErrCode(Integer errCode) { + this.errCode = errCode; + } + + public String getErrDesc() { + return errDesc; + } + + public void setErrDesc(String errDesc) { + this.errDesc = errDesc; + } + + public String getExecuteApplicationName() { + return executeApplicationName; + } + + public void setExecuteApplicationName(String executeApplicationName) { + this.executeApplicationName = executeApplicationName; + } + + public String getRequestApplicationName() { + return requestApplicationName; + } + + public void setRequestApplicationName(String requestApplicationName) { + this.requestApplicationName = requestApplicationName; + } + + public String getRunType() { + return runType; + } + + public void setRunType(String runType) { + this.runType = runType; + } + + public String getParamsJson() { + return paramsJson; + } + + public void setParamsJson(String paramsJson) { + this.paramsJson = paramsJson; + } + + public Long getCostTime() { + return costTime; + } + + public void setCostTime(Long costTime) { + this.costTime = costTime; + } + + public String getStrongerExecId() { + return strongerExecId; + } + + public void setStrongerExecId(String strongerExecId) { + this.strongerExecId = strongerExecId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/enums/ExecuteStrategyEnum.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/enums/ExecuteStrategyEnum.java new file mode 100644 index 000000000..729826b5d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/enums/ExecuteStrategyEnum.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.enums; + +import java.util.Arrays; + +public enum ExecuteStrategyEnum { + + + IS_EXECUTE("isExecute", "execute", "执行"), + IS_RE_EXECUTE("isReExecute", "reExecute", "失败重跑"), + IS_SELECTED_EXECUTE("isSelectedExecute", "selectedExecute", "选中执行"); + + private String name; + private String value; + private String desc; + + ExecuteStrategyEnum(String name, String value, String desc) { + this.name = name; + this.value = value; + this.desc = desc; + } + + public static ExecuteStrategyEnum getEnum(String value) { + return Arrays.stream(ExecuteStrategyEnum.values()).filter(e -> e.getValue().equals(value)).findFirst().orElseThrow(NullPointerException::new); + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppConnLinkisJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppConnLinkisJob.java new file mode 100644 index 000000000..7b2343985 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppConnLinkisJob.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job; + +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.job.AbstractAppConnLinkisJob; + + + +public class FlowExecutionAppConnLinkisJob extends AbstractAppConnLinkisJob { + + + @Override + public String getSubmitUser() { + return getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_SUBMIT_USER()); + } + + + + @Override + public String getUser() { + return getSubmitUser(); + //return getJobProps().get(FlowExecutionEntranceConfiguration.PROXY_USER()); + } + + @Override + public String getJobName() { + return getJobProps().get(FlowExecutionEntranceConfiguration.JOB_ID()); + } + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisJob.java new file mode 100644 index 000000000..d06f64f8e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisJob.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job; + +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; +import com.webank.wedatasphere.dss.linkis.node.execution.job.AbstractCommonLinkisJob; +import com.webank.wedatasphere.dss.linkis.node.execution.job.CommonLinkisJob; +import com.webank.wedatasphere.dss.linkis.node.execution.job.JobTypeEnum; +import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.ujes.client.response.JobExecuteResult; + +import java.util.ArrayList; +import java.util.Map; + + +public class FlowExecutionCommonLinkisJob extends AbstractCommonLinkisJob { + + + @Override + public String getSubmitUser() { + return getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_SUBMIT_USER()); + } + + + @Override + public String getUser() { + return getSubmitUser(); + //return getJobProps().get(FlowExecutionEntranceConfiguration.PROXY_USER()); + } + + @Override + public String getJobName() { + return getJobProps().get(FlowExecutionEntranceConfiguration.JOB_ID()); + } + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/persistence/WorkflowPersistenceEngine.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/persistence/WorkflowPersistenceEngine.java new file mode 100644 index 000000000..20b2086a1 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/persistence/WorkflowPersistenceEngine.java @@ -0,0 +1,227 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.flow.execution.entrance.persistence; + +import com.webank.wedatasphere.dss.flow.execution.entrance.service.WorkflowQueryService; +import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.entrance.exception.EntranceIllegalParamException; +import org.apache.linkis.entrance.exception.EntranceRPCException; +import org.apache.linkis.entrance.exception.QueryFailedException; +import org.apache.linkis.entrance.persistence.AbstractPersistenceEngine; +import org.apache.linkis.governance.common.entity.job.JobRequest; +import org.apache.linkis.governance.common.entity.job.SubJobDetail; +import org.apache.linkis.governance.common.entity.job.SubJobInfo; +import org.apache.linkis.governance.common.entity.task.*; +import org.apache.linkis.protocol.constants.TaskConstant; +import org.apache.linkis.protocol.task.Task; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + +public class WorkflowPersistenceEngine extends AbstractPersistenceEngine { + + @Autowired + private WorkflowQueryService workflowQueryService; + private static final Logger logger = LoggerFactory.getLogger(WorkflowPersistenceEngine.class); + + { + logger.info("WorkflowPersistenceEngine Registered"); + } + + public WorkflowPersistenceEngine() { + + } + + + @Override + public void persist(JobRequest jobRequest) throws QueryFailedException, EntranceIllegalParamException, EntranceRPCException { + + if (jobRequest == null) { + throw new EntranceIllegalParamException(20004, "task can not be null, unable to do persist operation"); + } + RequestInsertTask requestInsertTask = new RequestInsertTask(); + RequestPersistTask requestPersistTask = FlowExecutionUtils.jobRequest2RequestPersistTask(jobRequest); + BeanUtils.copyProperties(requestPersistTask, requestInsertTask); + ResponsePersist responsePersist = null; + try { + responsePersist = workflowQueryService.add(requestInsertTask,jobRequest); + } catch (Exception e) { + throw new EntranceRPCException(20020, "sender rpc failed", e); + } + + if (responsePersist != null) { + int status = responsePersist.getStatus(); + String message = responsePersist.getMsg(); + if (status != 0) { + throw new QueryFailedException(20011, "insert task failed, reason: " + message); + } + Map data = responsePersist.getData(); + Object object = data.get(TaskConstant.TASKID); + if (object == null) { + throw new QueryFailedException(20011, "insert task failed, reason: " + message); + } + String taskStr = object.toString(); + Long taskID = Long.parseLong(taskStr); + jobRequest.setId(taskID); + + } else { + //Todo can be throw exception if it is not requestPersistTask(todo 如果不是 requestPersistTask的话,可以进行throw异常) + } + } + + + @Override + public void persist(SubJobInfo subjobInfo) throws ErrorException { + + } + + // @Override +// public Task retrieve(Long taskID)throws EntranceIllegalParamException, QueryFailedException, EntranceRPCException { +// +// Task task = null; +// +// if ( taskID == null || taskID < 0){ +// throw new EntranceIllegalParamException(20003, "taskID can't be null or less than 0"); +// } +// +// RequestQueryTask requestQueryTask = new RequestQueryTask(); +// requestQueryTask.setTaskID(taskID); +// ResponsePersist responsePersist = null; +// try { +//// responsePersist = (ResponsePersist) sender.ask(requestQueryTask); +// responsePersist = workflowQueryService.query(requestQueryTask); +// }catch(Exception e){ +// logger.error("Requesting the corresponding task failed with taskID: {}(通过taskID: {} 请求相应的task失败)", taskID, e); +// throw new EntranceRPCException(20020, "sender rpc failed", e); +// } +// int status = responsePersist.getStatus(); +// //todo I want to discuss it again.(要再讨论下) +// String message = responsePersist.getMsg(); +// if (status != 0){ +// logger.error("By taskID: {} request the corresponding task return status code is not 0, the query fails(通过taskID: {} 请求相应的task返回状态码不为0,查询失败)", taskID); +// throw new QueryFailedException(20010, "retrieve task failed, reason: " + message); +// } +// Map data = responsePersist.getData(); +// if (data != null){ +// Object object = data.get(TaskConstant.TASK); +// if (object instanceof List){ +// List list = (List)object; +// if (list.size() == 0){ +// logger.info("returned list length is 0, maybe there is no task corresponding to {}", taskID); +// }else if (list.size() == 1){ +// Object t = list.get(0); +// Gson gson = new Gson(); +// String json = gson.toJson(t); +// task = gson.fromJson(json, RequestPersistTask.class); +// } +// } +// } +// return task; +// } + + + @Override + public void updateIfNeeded(JobRequest jobRequest) throws EntranceRPCException, EntranceIllegalParamException { + if (jobRequest == null) { + throw new EntranceIllegalParamException(20004, "task can not be null, unable to do update operation"); + } + + RequestUpdateTask requestUpdateTask = new RequestUpdateTask(); + RequestPersistTask requestPersistTask = FlowExecutionUtils.jobRequest2RequestPersistTask(jobRequest); + BeanUtils.copyProperties(requestPersistTask, requestUpdateTask); + try { + workflowQueryService.change(requestUpdateTask); + } catch (Exception e) { + logger.error("Request to update task with taskID {} failed, possibly due to RPC failure(请求更新taskID为 {} 的任务失败,原因可能是RPC失败)", requestUpdateTask.getTaskID(), e); + throw new EntranceRPCException(20020, "sender rpc failed ", e); + } + + } + + @Override + public void updateIfNeeded(SubJobInfo subJobInfo) throws ErrorException { + + } + + @Override + public Task[] readAll(String instance) throws EntranceIllegalParamException, EntranceRPCException, QueryFailedException { + + List retList = new ArrayList<>(); + + if (instance == null || "".equals(instance)) { + throw new EntranceIllegalParamException(20004, "instance can not be null"); + } + + RequestReadAllTask requestReadAllTask = new RequestReadAllTask(instance); + ResponsePersist responsePersist = null; + try { + //todo 目前没有使用到 +// responsePersist = (ResponsePersist)sender.ask(requestReadAllTask); +// workflowQueryService. + } catch (Exception e) { + throw new EntranceRPCException(20020, "sender rpc failed ", e); + } + if (responsePersist != null) { + int status = responsePersist.getStatus(); + String message = responsePersist.getMsg(); + if (status != 0) { + throw new QueryFailedException(20011, "read all tasks failed, reason: " + message); + } + Map data = responsePersist.getData(); + Object object = data.get(TaskConstant.TASK); + if (object instanceof List) { + List list = (List) object; + if (list.size() == 0) { + logger.info("no running task in this instance: {}", instance); + } + for (Object o : list) { + if (o instanceof RequestPersistTask) { + retList.add((RequestPersistTask) o); + } + } + } + } + return retList.toArray(new Task[0]); + } + + @Override + public JobRequest retrieveJobReq(Long jobGroupId) throws ErrorException { + return null; + } + + @Override + public SubJobDetail retrieveJobDetailReq(Long jobDetailId) throws ErrorException { + return null; + } + + @Override + public void close() throws IOException { + + } + + @Override + public void flush() throws IOException { + + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowEntranceRestfulApi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowEntranceRestfulApi.java new file mode 100644 index 000000000..215bbe9d9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowEntranceRestfulApi.java @@ -0,0 +1,224 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.restful; + +import com.webank.wedatasphere.dss.common.entity.DSSWorkspace; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.flow.execution.entrance.dao.TaskMapper; +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.WorkflowQueryTask; +import com.webank.wedatasphere.dss.flow.execution.entrance.enums.ExecuteStrategyEnum; +import com.webank.wedatasphere.dss.flow.execution.entrance.service.WorkflowExecutionInfoService; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.log.LogUtils; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.entrance.execute.EntranceJob; +import org.apache.linkis.entrance.restful.EntranceRestfulApi; +import org.apache.linkis.entrance.utils.JobHistoryHelper; +import org.apache.linkis.governance.common.entity.job.JobRequest; +import org.apache.linkis.protocol.constants.TaskConstant; +import org.apache.linkis.protocol.utils.ZuulEntranceUtils; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.queue.Job; +import org.apache.linkis.scheduler.queue.SchedulerEventState; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import scala.Option; + +import javax.servlet.http.HttpServletRequest; +import java.util.List; +import java.util.Map; +import java.util.Optional; + + +@RequestMapping(path = "/dss/flow/entrance") +@RestController +public class FlowEntranceRestfulApi extends EntranceRestfulApi { + + private EntranceServer entranceServer; + + private static final Logger logger = LoggerFactory.getLogger(FlowEntranceRestfulApi.class); + + @Autowired + private WorkflowExecutionInfoService workflowExecutionInfoService; + @Autowired + private TaskMapper taskMapper; + @Override + @Autowired + public void setEntranceServer(EntranceServer entranceServer) { + super.setEntranceServer(entranceServer); + this.entranceServer = entranceServer; + } + + /** + * The execute function handles the request submitted by the user to execute the task, and the execution ID is returned to the user. + * execute函数处理的是用户提交执行任务的请求,返回给用户的是执行ID + * json Incoming key-value pair(传入的键值对) + * Repsonse + */ + @Override + @RequestMapping(value = "/execute",method = RequestMethod.POST) + public Message execute(HttpServletRequest req, @RequestBody Map json) { + Message message = null; +// try{ + logger.info("Begin to get an execID"); + DSSWorkspace workspace = SSOHelper.getWorkspace(req); + String umUser = SecurityFilter.getLoginUsername(req); + json.put(TaskConstant.UMUSER, umUser); + json.put(TaskConstant.EXECUTE_USER, umUser); + Map params = (Map) json.get("params"); + //失败节点重新执行功能,通过isReExecute字段来判断 + if (json.containsKey(ExecuteStrategyEnum.IS_RE_EXECUTE.getName())) { + params.put("executeStrategy", ExecuteStrategyEnum.IS_RE_EXECUTE.getValue()); + try { + String flowIdStr = BDPJettyServerHelper.jacksonJson().readValue(json.get("executionCode").toString(), Map.class).get("flowId").toString(); + String succeedJobIdsStr = workflowExecutionInfoService.getSucceedJobsByFlowId(Long.parseLong(flowIdStr)); + if (StringUtils.isNotEmpty(succeedJobIdsStr)) { + params.put("nodeID", succeedJobIdsStr); + } + } catch (Exception e) { + logger.error("re execute job failed:", e); + } + } else if (json.containsKey(ExecuteStrategyEnum.IS_SELECTED_EXECUTE.getName())) { + //选中节点执行功能,通过isSelectedExecute字段来判断 + params.put("executeStrategy", ExecuteStrategyEnum.IS_SELECTED_EXECUTE.getValue()); + String nodeID = Optional.ofNullable(json.get("nodeID").toString()).orElseThrow(() -> { + logger.error("execute selected node failed because nodeID is empty"); + return new NullPointerException(); + }); + params.put("nodeID", nodeID); + } else { + //默认为执行功能 + params.put("executeStrategy", ExecuteStrategyEnum.IS_EXECUTE.getValue()); + } + params.put("workspace", workspace); + String label = ((Map) json.get(DSSCommonUtils.DSS_LABELS_KEY)).get("route").toString(); + params.put(DSSCommonUtils.DSS_LABELS_KEY, label); + String execID = entranceServer.execute(json); + Job job = entranceServer.getJob(execID).get(); + JobRequest task = ((EntranceJob) job).getJobRequest(); + Long taskID = task.getId(); + pushLog(LogUtils.generateInfo("You have submitted a new job, script code (after variable substitution) is"), job); + pushLog("************************************SCRIPT CODE************************************", job); + pushLog(task.getExecutionCode(), job); + pushLog("************************************SCRIPT CODE************************************", job); + pushLog(LogUtils.generateInfo("Your job is accepted, jobID is " + execID + " and taskID is " + taskID + ". Please wait it to be scheduled"), job); + execID = ZuulEntranceUtils.generateExecID(execID, Sender.getThisServiceInstance().getApplicationName(), new String[]{Sender.getThisInstance()}); + message = Message.ok(); + message.setMethod("/api/entrance/execute"); + message.data("execID", execID); + message.data("taskID", taskID); + logger.info("End to get an an execID: {}, taskID: {}", execID, taskID); + return message; + } + + @Override + @RequestMapping(value = "/{id}/status",method = RequestMethod.GET) + public Message status(@PathVariable("id") String id, @RequestParam(required = false, name = "taskID") String taskID) { + Message message = null; + String realId = ZuulEntranceUtils.parseExecID(id)[3]; + Option job; + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.warn("获取任务 {} 状态时出现错误", realId, e); + //如果获取错误了,证明在内存中已经没有了,去flow task表里面找寻一下taskID代表的任务的状态,然后返回 + long realTaskID = Long.parseLong(taskID); + WorkflowQueryTask queryTask = new WorkflowQueryTask(); + queryTask.setTaskID(realTaskID); + List taskList = taskMapper.selectTask(queryTask); + message = Message.ok(); + if (null == taskList || taskList.size() != 1) { + int size = 0; + if (null != taskList) { + size = taskList.size(); + } + logger.error("Got {} task for taskID : {}", size, realTaskID); + message = Message.error("Got " + size + " task for taskId : " + realTaskID); + } else { + String status = taskList.get(0).getStatus(); + message.data("status", status).data("execID", id); + } + message.setMethod("/api/entrance/" + id + "/status"); + return message; + } + if (job.isDefined()) { + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", job.get().getState().toString()).data("execID", id); + } else { + message = Message.error("ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空,不能获取相应的任务状态)"); + } + return message; + } + + @RequestMapping(path = {"/{id}/kill"},method = {RequestMethod.GET}) + public Message kill(@PathVariable("id") String id, @RequestParam(value = "taskID",required = false) Long taskID) { + String realId = ZuulEntranceUtils.parseExecID(id)[3]; + Option job; + try { + job = this.entranceServer.getJob(realId); + } catch (Exception var10) { + logger.warn("can not find a job in entranceServer, will force to kill it", var10); + JobHistoryHelper.forceKill(taskID); + Message message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(0); + return message; + } + Message message; + if (job.isEmpty()) { + logger.warn("can not find a job in entranceServer, will force to kill it"); + JobHistoryHelper.forceKill(taskID); + message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(0); + return message; + } else { + try { + logger.info("begin to kill job {} ", ((Job)job.get()).getId()); + ((Job)job.get()).kill(); + message = Message.ok("Successfully killed the job(成功kill了job)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(0); + message.data("execID", id); + if (job.get() instanceof EntranceJob) { + EntranceJob entranceJob = (EntranceJob)job.get(); + JobRequest jobReq = entranceJob.getJobRequest(); + entranceJob.updateJobRequestStatus(SchedulerEventState.Cancelled().toString()); + this.entranceServer.getEntranceContext().getOrCreatePersistenceManager().createPersistenceEngine().updateIfNeeded(jobReq); + } + logger.info("end to kill job {} ", ((Job)job.get()).getId()); + } catch (Throwable var9) { + logger.error("kill job {} failed ", ((Job)job.get()).getId(), var9); + message = Message.error("An exception occurred while killing the job, kill failed(kill job的时候出现了异常,kill失败)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(1); + } + return message; + } + } + + private void pushLog(String log, Job job) { + entranceServer.getEntranceContext().getOrCreateLogManager().onLogUpdate(job, log); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowExecutionRestfulApi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowExecutionRestfulApi.java new file mode 100644 index 000000000..ae2893f5f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowExecutionRestfulApi.java @@ -0,0 +1,144 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.restful; + +import com.webank.wedatasphere.dss.flow.execution.entrance.FlowContext$; +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.WorkflowExecuteInfoVo; +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob; +import com.webank.wedatasphere.dss.flow.execution.entrance.service.WorkflowExecutionInfoService; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.protocol.utils.ZuulEntranceUtils; +import org.apache.linkis.scheduler.queue.Job; +import org.apache.linkis.scheduler.queue.SchedulerEventState; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import scala.Enumeration; +import scala.Option; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@RequestMapping(path = "/dss/flow/entrance") +@RestController +public class FlowExecutionRestfulApi { + + private EntranceServer entranceServer; + + private static final Logger logger = LoggerFactory.getLogger(FlowExecutionRestfulApi.class); + + @Autowired + public void setEntranceServer(EntranceServer entranceServer) { + this.entranceServer = entranceServer; + } + + @Autowired + private WorkflowExecutionInfoService workflowExecutionInfoService; + + @RequestMapping(value = "/{id}/execution",method = RequestMethod.GET) + public Message execution(@PathVariable("id") String id,@RequestParam("labels") String labels) { + Message message = null; + String realId = ZuulEntranceUtils.parseExecID(id)[3]; + Option job = entranceServer.getJob(realId); + try { + if (job.isDefined() && job.get() instanceof FlowEntranceJob) { + logger.info("Start to get job {} execution info", job.get().getId()); + FlowEntranceJob flowEntranceJob = (FlowEntranceJob) job.get(); + message = Message.ok("Successfully get job execution info"); + message.setMethod("/api/entrance/" + id + "/execution"); + message.setStatus(0); + long nowTime = System.currentTimeMillis(); + flowEntranceJob.getFlowContext().getRunningNodes().forEach((k, v) -> { + if (v != null) { + v.setNowTime(nowTime); + } + }); + List> runningJobsList = FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getRunningNodes()); + message.data("runningJobs", runningJobsList); + + List> pendingList = FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getPendingNodes()); + pendingList.addAll(FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getSkippedNodes())); + message.data("pendingJobs", pendingList); + + List> succeedJobsList = FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getSucceedNodes()); + message.data("succeedJobs", succeedJobsList); + + List> failedJobsList = FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getFailedNodes()); + message.data("failedJobs", failedJobsList); + + List> skippedJobsList = FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getSkippedNodes()); + message.data("skippedJobs", skippedJobsList); + + //如果执行失败或执行成功,将执行结果存储到数据库 + int status = getJobStatus(flowEntranceJob); + if (status < 2) { + WorkflowExecuteInfoVo workflowExecuteInfoVo = new WorkflowExecuteInfoVo(); + workflowExecuteInfoVo.setTaskId(flowEntranceJob.getJobRequest().getId()); + + String flowIdStr = BDPJettyServerHelper.jacksonJson().readValue(flowEntranceJob.getJobRequest().getExecutionCode(), Map.class).get("flowId").toString(); + String version = BDPJettyServerHelper.jacksonJson().readValue(flowEntranceJob.getJobRequest().getExecutionCode(), Map.class).get("version").toString(); + workflowExecuteInfoVo.setFlowId(Long.parseLong(flowIdStr)); + workflowExecuteInfoVo.setVersion(version); + + workflowExecuteInfoVo.setFailedJobsList(failedJobsList); + workflowExecuteInfoVo.setPendingJobsList(pendingList); + workflowExecuteInfoVo.setSkippedJobsList(skippedJobsList); + workflowExecuteInfoVo.setRunningJobsList(runningJobsList); + workflowExecuteInfoVo.setSucceedJobsList(succeedJobsList); + workflowExecuteInfoVo.setCreatetime(new Date()); + workflowExecuteInfoVo.setStatus(status); + workflowExecutionInfoService.saveExecuteInfo(workflowExecuteInfoVo); + } + } else { + message = Message.error("ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空,不能获取相应的任务状态)"); + } + } catch (Exception e) { + logger.error("Failed to get job execution info:", e); + message = Message.error("Failed to get job execution info:" + e.getMessage()); + } + return message; + } + + /** + * 获取job的状态 + * @param flowEntranceJob + * @return + */ + private int getJobStatus(FlowEntranceJob flowEntranceJob) { + Enumeration.Value state = flowEntranceJob.getState(); + int status = 2; + if(SchedulerEventState.Failed().equals(state) + || SchedulerEventState.Cancelled().equals(state) + || SchedulerEventState.Timeout().equals(state)){ + //失败 + status = 0; + }else if(SchedulerEventState.Succeed().equals(state)){ + //成功 + status = 1; + } + return status; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/WorkflowExecutionInfoService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/WorkflowExecutionInfoService.java new file mode 100644 index 000000000..f2efecbfd --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/WorkflowExecutionInfoService.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.service; + +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.WorkflowExecuteInfo; +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.WorkflowExecuteInfoVo; + +public interface WorkflowExecutionInfoService { + + WorkflowExecuteInfo getExecuteInfoByFlowId(Long flowId); + + void saveExecuteInfo(WorkflowExecuteInfoVo workflowExecuteInfoVo); + + String getSucceedJobsByFlowId(Long flowId); + +} + + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/WorkflowQueryService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/WorkflowQueryService.java new file mode 100644 index 000000000..ce887c252 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/WorkflowQueryService.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.service; + +import org.apache.linkis.governance.common.entity.job.JobRequest; +import org.apache.linkis.governance.common.entity.task.*; + + +public interface WorkflowQueryService { + + ResponsePersist add(RequestInsertTask requestInsertTask, JobRequest jobRequest); + + ResponsePersist change(RequestUpdateTask requestUpdateTask); + +// ResponsePersist query(RequestQueryTask requestQueryTask); +// +// WorkflowQueryTaskVO getTaskByID(Long taskID, String userName); +// +// List search(Long taskID, String username, String status, Date sDate, Date eDate, String executeApplicationName); +// +// RequestPersistTask searchOne(String instance, String execId, Date sDate, Date eDate, String executeApplicationName); +// +// List getQueryVOList(List list); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/impl/WorkflowExecutionInfoServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/impl/WorkflowExecutionInfoServiceImpl.java new file mode 100644 index 000000000..d30905b4c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/impl/WorkflowExecutionInfoServiceImpl.java @@ -0,0 +1,91 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.service.impl; + +import com.webank.wedatasphere.dss.flow.execution.entrance.dao.WorkflowExecuteInfoMapper; +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.WorkflowExecuteInfo; +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.WorkflowExecuteInfoVo; +import com.webank.wedatasphere.dss.flow.execution.entrance.service.WorkflowExecutionInfoService; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.stream.Collectors; + +@Service +public class WorkflowExecutionInfoServiceImpl implements WorkflowExecutionInfoService { + + @Autowired + private WorkflowExecuteInfoMapper workflowExecuteInfoMapper; + + @Override + public WorkflowExecuteInfo getExecuteInfoByFlowId(Long flowId) { + return workflowExecuteInfoMapper.getExecuteInfoByFlowId(flowId); + } + + @Override + public void saveExecuteInfo(WorkflowExecuteInfoVo workflowExecuteInfoVo) { + if (workflowExecuteInfoVo == null) { + return; + } + WorkflowExecuteInfo executeInfo = new WorkflowExecuteInfo(); + executeInfo.setTaskId(workflowExecuteInfoVo.getTaskId()); + executeInfo.setFlowId(workflowExecuteInfoVo.getFlowId()); + executeInfo.setVersion(workflowExecuteInfoVo.getVersion()); + executeInfo.setStatus(workflowExecuteInfoVo.getStatus()); + executeInfo.setCreatetime(workflowExecuteInfoVo.getCreatetime()); + executeInfo.setFailedJobs(workflowExecuteInfoVo.getFailedJobsList().stream().map(map -> map.get("nodeID").toString()).collect(Collectors.joining(","))); + executeInfo.setPendingJobs(workflowExecuteInfoVo.getPendingJobsList().stream().map(map -> map.get("nodeID").toString()).collect(Collectors.joining(","))); + executeInfo.setSkippedJobs(workflowExecuteInfoVo.getSkippedJobsList().stream().map(map -> map.get("nodeID").toString()).collect(Collectors.joining(","))); + executeInfo.setSucceedJobs(workflowExecuteInfoVo.getSucceedJobsList().stream().map(map -> map.get("nodeID").toString()).collect(Collectors.joining(","))); + executeInfo.setRunningJobs(workflowExecuteInfoVo.getRunningJobsList().stream().map(map -> map.get("nodeID").toString()).collect(Collectors.joining(","))); + + WorkflowExecuteInfo executeInfoDB = workflowExecuteInfoMapper.getExecuteInfoByFlowIdAndVersion(workflowExecuteInfoVo.getFlowId(), workflowExecuteInfoVo.getVersion()); + if (executeInfoDB != null) { + executeInfo.setId(executeInfoDB.getId()); + //成功的节点进行拼接 + executeInfo.setSucceedJobs(getSucceedJobsStr(executeInfo.getSucceedJobs(),executeInfoDB.getSucceedJobs())); + workflowExecuteInfoMapper.updateWorkflowExecuteInfo(executeInfo); + } else { + workflowExecuteInfoMapper.insert(executeInfo); + } + } + + public String getSucceedJobsStr(String sucStr,String sucStrDB) { + if(StringUtils.isBlank(sucStr) || StringUtils.isBlank(sucStrDB)){ + return StringUtils.isNotBlank(sucStr) ? sucStr : sucStrDB; + } + String[] nodeListArr = sucStrDB.split(","); + for (int i = 0; i < nodeListArr.length; i++) { + String nodeId = nodeListArr[i]; + if (StringUtils.isBlank(nodeId) || sucStr.contains(nodeId)) { + continue; + } + if(!sucStr.endsWith(",")){ + sucStr = sucStr.concat(","); + } + sucStr = sucStr.concat(nodeId); + } + if(sucStr.endsWith(",")){ + sucStr = sucStr.substring(0,sucStr.length()-1); + } + return sucStr; + } + + @Override + public String getSucceedJobsByFlowId(Long flowId) { + WorkflowExecuteInfo executeInfo = getExecuteInfoByFlowId(flowId); + if (executeInfo != null) { + String skippedJobs = executeInfo.getSkippedJobs(); + String succeedJobs = executeInfo.getSucceedJobs(); + //如果skippedjobs有内容,也需要添加上 + if (StringUtils.isNotEmpty(skippedJobs)) { + if (StringUtils.isNotEmpty(succeedJobs)) { + return succeedJobs + "," + skippedJobs; + } + return skippedJobs; + } + return succeedJobs; + } + return null; + } +} + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/impl/WorkflowQueryServiceImpl.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/impl/WorkflowQueryServiceImpl.scala new file mode 100644 index 000000000..0bc687aa5 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/service/impl/WorkflowQueryServiceImpl.scala @@ -0,0 +1,242 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.service.impl + +import java.lang.Long +import java.util +import java.util.Date + +import com.google.common.collect.Iterables +import com.webank.wedatasphere.dss.flow.execution.entrance.dao.TaskMapper +import com.webank.wedatasphere.dss.flow.execution.entrance.entity.{WorkflowQueryTask, WorkflowQueryTaskVO} +import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowQueryErrorException +import com.webank.wedatasphere.dss.flow.execution.entrance.service.WorkflowQueryService +import com.webank.wedatasphere.dss.flow.execution.entrance.status.TaskStatus +import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.entity.task._ +import org.apache.linkis.manager.label.utils.LabelUtils +import org.apache.linkis.protocol.utils.ZuulEntranceUtils +import org.apache.commons.lang3.StringUtils +import org.springframework.beans.BeanUtils +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional + + +@Service +class WorkflowQueryServiceImpl extends WorkflowQueryService with Logging { + @Autowired + private var taskMapper: TaskMapper = _ +// @Autowired +// private var queryCacheService: QueryCacheService = _ + + override def add(requestInsertTask: RequestInsertTask,jobRequest: JobRequest): ResponsePersist = { + info("Insert data into the database(往数据库中插入数据):" + requestInsertTask.toString) +// QueryUtils.storeExecutionCode(requestInsertTask) + val persist = new ResponsePersist + Utils.tryCatch { + val queryTask = requestPersistTaskTask2QueryTask(requestInsertTask) + val insertEntity = new WorkflowQueryTask(); + BeanUtils.copyProperties(queryTask,insertEntity) + insertEntity.setExecutionCode(jobRequest.getExecutionCode) + taskMapper.insertTask(insertEntity) + val map = new util.HashMap[String, Object]() + map.put("taskID", insertEntity.getTaskID()) + persist.setStatus(0) + persist.setData(map) + } { + case e: Exception => + error(e.getMessage) + persist.setStatus(1) + persist.setMsg(e.getMessage) + } + persist + } + + @Transactional + override def change(requestUpdateTask: RequestUpdateTask): ResponsePersist = { + val executionCode = requestUpdateTask.getExecutionCode + requestUpdateTask.setExecutionCode(null) + info("Update data to the database(往数据库中更新数据):" + requestUpdateTask.toString) + /*//TODO 去查数据内容 + if (StringUtils.isNotEmpty(requestUpdateTask.getEngineInstance)) { + requestUpdateTask.setTaskResource(taskMapper.getResource(requestUpdateTask.getEngineInstance)); + }*/ + val persist = new ResponsePersist + Utils.tryCatch { + if (requestUpdateTask.getErrDesc != null) { + if (requestUpdateTask.getErrDesc.length > 256) { + info(s"errorDesc is too long,we will cut some message") + requestUpdateTask.setErrDesc(requestUpdateTask.getErrDesc.substring(0, 256)) + info(s"${requestUpdateTask.getErrDesc}") + } + } + if (requestUpdateTask.getStatus != null) { + val oldStatus: String = taskMapper.selectTaskStatusForUpdate(requestUpdateTask.getTaskID) + if (oldStatus != null && !shouldUpdate(oldStatus, requestUpdateTask.getStatus)) + throw new FlowQueryErrorException(100639,s"${requestUpdateTask.getTaskID}数据库中的task状态为:${oldStatus}更新的task状态为:${requestUpdateTask.getStatus}更新失败!") + } + val updateTask = requestPersistTaskTask2QueryTask(requestUpdateTask) + updateTask.setUpdatedTime(new Date()) + taskMapper.updateTask(updateTask) + + //updated by shanhuang to write cache + //todo add cache + if (TaskStatus.Succeed.toString.equals(requestUpdateTask.getStatus)) { + info("Write cache for task: " + requestUpdateTask.getTaskID) + requestUpdateTask.setExecutionCode(executionCode) +// queryCacheService.writeCache(requestUpdateTask) + } + + val map = new util.HashMap[String, Object] + map.put("taskID", requestUpdateTask.getTaskID) + persist.setStatus(0) + persist.setData(map) + } { + case e: Exception => + error(e.getMessage) + persist.setStatus(1); + persist.setMsg(e.getMessage); + } + persist + } + + def query(requestQueryTask: RequestQueryTask): ResponsePersist = { + info("查询历史task:" + requestQueryTask.toString) + val persist = new ResponsePersist + Utils.tryCatch { + val task = taskMapper.selectTask(requestPersistTaskTask2QueryTask(requestQueryTask)) + val map = new util.HashMap[String, Object]() + map.put("task", queryTaskList2RequestPersistTaskList(task)) + persist.setStatus(0) + persist.setData(map) + } { + case e: Exception => + error(e.getMessage) + persist.setStatus(1); + persist.setMsg(e.getMessage); + } + persist + } + + private def queryTaskList2RequestPersistTaskList(queryTask: java.util.List[WorkflowQueryTask]): java.util.List[RequestPersistTask] = { + import scala.collection.JavaConversions._ + val tasks = new util.ArrayList[RequestPersistTask] + queryTask.foreach(f => tasks.add(queryTask2RequestPersistTask(f))) + tasks + } + + def queryTask2RequestPersistTask(queryTask: WorkflowQueryTask): RequestPersistTask = { +// QueryUtils.exchangeExecutionCode(queryTask) + val task = new RequestPersistTask + BeanUtils.copyProperties(queryTask, task) + task.setSource(FlowExecutionUtils.gson.fromJson(queryTask.getSourceJson, classOf[java.util.HashMap[String, String]])) + task.setParams(FlowExecutionUtils.gson.fromJson(queryTask.getParamsJson, classOf[java.util.HashMap[String, Object]])) + task + } + + private def requestPersistTaskTask2QueryTask(requestPersistTask: RequestPersistTask): WorkflowQueryTask = { + val task: WorkflowQueryTask = new WorkflowQueryTask + BeanUtils.copyProperties(requestPersistTask, task) + if (requestPersistTask.getSource != null) + task.setSourceJson(FlowExecutionUtils.gson.toJson(requestPersistTask.getSource)) + if (requestPersistTask.getParams != null) + task.setParamsJson(FlowExecutionUtils.gson.toJson(requestPersistTask.getParams)) + if (requestPersistTask.getLabels != null) + task.setLabelJson(FlowExecutionUtils.gson.toJson(LabelUtils.labelsToMap(requestPersistTask.getLabels))) + task + } + +// override def getTaskByID(taskID: Long, userName: String): WorkflowQueryTaskVO = { +// val task = new WorkflowQueryTask +// task.setTaskID(taskID) +// task.setUmUser(userName) +// val taskR = taskMapper.selectTask(task) +// import scala.collection.JavaConversions._ +// if (taskR.isEmpty) null else queryTask2QueryTaskVO(taskR(0)) +// } + + + def queryTask2QueryTaskVO(queryTask: WorkflowQueryTask): WorkflowQueryTaskVO = { +// QueryUtils.exchangeExecutionCode(queryTask) + val taskVO = new WorkflowQueryTaskVO + BeanUtils.copyProperties(queryTask, taskVO) + if (!StringUtils.isEmpty(taskVO.getSourceJson)) { + Utils.tryCatch { + val source = FlowExecutionUtils.gson.fromJson(taskVO.getSourceJson, classOf[util.Map[String, String]]) + import scala.collection.JavaConversions._ + taskVO.setSourceTailor(source.map(_._2).foldLeft("")(_ + _ + "-").stripSuffix("-")) + } { + case _ => warn("sourceJson deserializae failed,this task may be the old data") + } + } + if (queryTask.getExecId() != null && queryTask.getExecuteApplicationName() != null && queryTask.getInstance() != null) { + taskVO.setStrongerExecId(ZuulEntranceUtils.generateExecID(queryTask.getExecId(), + queryTask.getExecuteApplicationName(), queryTask.getInstance(), queryTask.getRequestApplicationName)) + } + val status = queryTask.getStatus() + val createdTime = queryTask.getCreatedTime() + val updatedTime = queryTask.getUpdatedTime() + if (isJobFinished(status) && createdTime != null && updatedTime != null) { + taskVO.setCostTime(queryTask.getUpdatedTime().getTime() - queryTask.getCreatedTime().getTime()); + } else if (createdTime != null) { + taskVO.setCostTime(System.currentTimeMillis() - queryTask.getCreatedTime().getTime()); + } + taskVO + } + + def isJobFinished(status: String): Boolean = { + TaskStatus.Succeed.toString.equals(status) || + TaskStatus.Failed.toString.equals(status) || + TaskStatus.Cancelled.toString.equals(status) || + TaskStatus.Timeout.toString.equals(status) + } + +// override def search(taskID: Long, username: String, status: String, sDate: Date, eDate: Date, executeApplicationName: String): util.List[WorkflowQueryTask] = { +// import scala.collection.JavaConversions._ +// val split: util.List[String] = if (status != null) status.split(",").toList else null +// taskMapper.search(taskID, username, split, sDate, eDate, executeApplicationName, null, null) +// } + + def getQueryVOList(list: java.util.List[WorkflowQueryTask]): java.util.List[WorkflowQueryTaskVO] = { + val ovs = new util.ArrayList[WorkflowQueryTaskVO] + import scala.collection.JavaConversions._ + list.foreach(f => { + ovs.add(queryTask2QueryTaskVO(f)) + }) + ovs + } + + private def shouldUpdate(oldStatus: String, newStatus: String): Boolean = TaskStatus.valueOf(oldStatus).ordinal <= TaskStatus.valueOf(newStatus).ordinal + +// override def searchOne(instance: String, execId: String, sDate: Date, eDate: Date, executeApplicationName: String): RequestPersistTask = { +// Iterables.getFirst( +// queryTaskList2RequestPersistTaskList(taskMapper.search(null, null, null, sDate, eDate, executeApplicationName, instance, execId)), +// { +// val requestPersistTask = new RequestPersistTask +// requestPersistTask.setInstance(instance) +// requestPersistTask.setExecId(execId) +// requestPersistTask.setStatus(TaskStatus.Inited.toString) +// requestPersistTask.setProgress(0.0f) +// requestPersistTask.setTaskID(0l) +// requestPersistTask +// }) +// } +} + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/status/TaskStatus.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/status/TaskStatus.java new file mode 100644 index 000000000..f783999ee --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/status/TaskStatus.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.status; + + +public enum TaskStatus { + Inited, WaitForRetry, Scheduled, Running, Succeed, Failed, Cancelled, Timeout +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/DSSFowExecutionServerApplication.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/DSSFowExecutionServerApplication.scala new file mode 100644 index 000000000..bfa297d9c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/DSSFowExecutionServerApplication.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + + +object DSSFowExecutionServerApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContext.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContext.scala new file mode 100644 index 000000000..a02644896 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContext.scala @@ -0,0 +1,84 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance + +import java.util + +import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeRunner +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode +import org.apache.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState + + +trait FlowContext { + + def getRunningNodes: util.Map[String, NodeRunner] + + def getSucceedNodes: util.Map[String, NodeRunner] + + def getPendingNodes: util.Map[String, NodeRunner] + + def getFailedNodes: util.Map[String, NodeRunner] + + def getSkippedNodes: util.Map[String, NodeRunner] + + def getScheduledNodes: util.Map[String, NodeRunner] + + def getFlowStatus: SchedulerEventState + + def isNodeCompleted(nodeName: String): Boolean +} + +object FlowContext { + + def isNodeRunning(nodeName: String, flowContext: FlowContext):Boolean = { + flowContext.getRunningNodes.containsKey(nodeName) || flowContext.getScheduledNodes.containsKey(nodeName) + } + + def changedNodeState(fromMap: util.Map[String, NodeRunner], + toMap: util.Map[String, NodeRunner], node: WorkflowNode, info:String): Unit = { + val nodeName = node.getDSSNode.getName + if (fromMap.containsKey(nodeName)) { + val runner = fromMap.get(nodeName) + runner.setNodeExecutedInfo(info) + toMap.put(nodeName, runner) + fromMap.remove(nodeName) + } + } + + def convertView(nodeRunnerMap: java.util.Map[String, NodeRunner]): java.util.List[java.util.Map[String, Any]] = { + val nodes = new util.ArrayList[java.util.Map[String, Any]]() + val iterator = nodeRunnerMap.entrySet().iterator() + while (iterator.hasNext) { + val nodeRunner = iterator.next().getValue + if (nodeRunner != null && nodeRunner.getNode != null) { + val nodeView = new java.util.HashMap[String, Any]() + val linkisJob = nodeRunner.getLinkisJob + val node = nodeRunner.getNode + if (linkisJob != null && linkisJob.getJobExecuteResult != null) { + nodeView.put("execID", linkisJob.getJobExecuteResult.getExecID) + nodeView.put("taskID", linkisJob.getJobExecuteResult.getTaskID()) + } + nodeView.put("nodeID", node.getId) + nodeView.put("info", nodeRunner.getNodeExecutedInfo()) + nodeView.put("startTime", nodeRunner.getStartTime()) + nodeView.put("nowTime", nodeRunner.getNowTime()) + nodes.add(nodeView) + } + } + nodes + } +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContextImpl.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContextImpl.scala new file mode 100644 index 000000000..f02cc5d9e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContextImpl.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance + +import java.util + +import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeRunner +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.scheduler.queue.SchedulerEventState +import org.apache.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState + + +class FlowContextImpl extends FlowContext with Logging { + + val runningNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] + + val succeedNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] + + val pendingNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] + + val failedNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] + + val skippedNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] + + val scheduledNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] + + val flowStatus:SchedulerEventState = SchedulerEventState.Inited + + override def getRunningNodes: util.Map[String, NodeRunner] = this.runningNodes + + override def getSucceedNodes: util.Map[String, NodeRunner] = this.succeedNodes + + override def getPendingNodes: util.Map[String, NodeRunner] =this.pendingNodes + + override def getFailedNodes: util.Map[String, NodeRunner] = this.failedNodes + + override def getSkippedNodes: util.Map[String, NodeRunner] = this.skippedNodes + + override def getScheduledNodes: util.Map[String, NodeRunner] = this.scheduledNodes + + override def getFlowStatus: SchedulerEventState = this.flowStatus + + override def isNodeCompleted(nodeName: String): Boolean = { + getSkippedNodes.containsKey(nodeName) || getSucceedNodes.containsKey(nodeName) || getFailedNodes.containsKey(nodeName) + } + +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala new file mode 100644 index 000000000..bbd41a519 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala @@ -0,0 +1,76 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.conf + +import org.apache.linkis.common.conf.CommonVars + + +object FlowExecutionEntranceConfiguration { + + val WORKFLOW_APPLICATION_NAME = CommonVars("wds.dss.workflow.service.name", "DSS-WORKFLOW-SERVER-") + + val NODE_PRINT_FLAG = CommonVars("wds.linkis.flow.node.log.flag", false) + + val NODE_STATUS_POLLER_THREAD_SIZE = CommonVars("wds.dds.flow.node.status.poller.thread.size", 20) + + val NODE_STATUS_POLLER_SCHEDULER_TIME = CommonVars("wds.dds.flow.node.status.poller.scheduler.time", 5) + val FLOW_EXECUTION_SCHEDULER_POOL_SIZE = CommonVars("wds.linkis.flow.execution.pool.size", 30) + + val NODE_STATUS_INTERVAL = CommonVars("wds.dds.flow.node.status.poller.interval.time", 3000) + + val COMMAND = "command" + + val JOB_ID = "job.id" + + val FLOW_NAME = "dss.flow.flowid" + + + val PROJECT_NAME = "dss.flow.projectname" + + val FLOW_EXEC_ID = "dss.flow.execid" + + val WORKSPACE_ID = "dss.flow.workspace.id" + + + val PROXY_USER = "user.to.proxy" + + + val FLOW_SUBMIT_USER = "flow.submituser" + + + val SKIP_NODES = CommonVars("wds.dss.flow.skip.nodes", "workflow.subflow") + + val SIGNAL_NODES = CommonVars("wds.dss.flow.signal.nodes", "linkis.appconn.eventchecker.eventreceiver") + + val PROPS_MAP = "props_map" + + val WORKSPACE = "workspace" + + val CONTEXT_ID = "contextID" + + val NODE_NAME = "nodeName" + + val FLOW_VAR_MAP = "flow_var_map" + + + val NODE_CONFIGURATION_KEY = "configuration" + + val PROJECT_RESOURCES = "project_resources" + + val FLOW_RESOURCES = "flow_resources" + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceSpringConfiguration.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceSpringConfiguration.scala new file mode 100644 index 000000000..6c74054a0 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceSpringConfiguration.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.conf + +import com.webank.wedatasphere.dss.flow.execution.entrance.engine.{FlowEntranceEngine, FlowExecutionExecutorManagerImpl} +import com.webank.wedatasphere.dss.flow.execution.entrance.entranceparser.FlowExecutionParser +import com.webank.wedatasphere.dss.flow.execution.entrance.persistence.WorkflowPersistenceEngine +import org.apache.linkis.entrance.EntranceParser +import org.apache.linkis.entrance.persistence.{PersistenceEngine, PersistenceManager} +import org.apache.linkis.scheduler.executer.ExecutorManager +import org.slf4j.LoggerFactory +import org.springframework.context.annotation.{Bean, Configuration} + +@Configuration +class FlowExecutionEntranceSpringConfiguration { + private val logger = LoggerFactory.getLogger(classOf[FlowExecutionEntranceSpringConfiguration]) + + @Bean + def executorManager(flowEntranceEngine: FlowEntranceEngine): ExecutorManager = { + logger.info("begin to get FlowExecution Entrance EntranceExecutorManager") + new FlowExecutionExecutorManagerImpl(flowEntranceEngine) + } + + @Bean + def persistenceEngine(): PersistenceEngine ={ + new WorkflowPersistenceEngine() + } + + @Bean + def entranceParser(persistenceManager: PersistenceManager): EntranceParser = { + logger.info("begin to get FlowExecution Entrance parser") + new FlowExecutionParser(persistenceManager) + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowEntranceEngine.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowEntranceEngine.scala new file mode 100644 index 000000000..06af62226 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowEntranceEngine.scala @@ -0,0 +1,129 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.engine + +import java.util + +import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException +import com.webank.wedatasphere.dss.flow.execution.entrance.execution.FlowExecution +import com.webank.wedatasphere.dss.flow.execution.entrance.job.parser.FlowEntranceJobParser +import com.webank.wedatasphere.dss.flow.execution.entrance.job.{FlowEntranceJob, FlowExecutionRequest} +import com.webank.wedatasphere.dss.flow.execution.entrance.resolver.FlowDependencyResolver +import org.apache.commons.lang.exception.ExceptionUtils +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.scheduler.executer.ExecutorState.ExecutorState +import org.apache.linkis.scheduler.executer._ +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +@Component +class FlowEntranceEngine extends Executor with ConcurrentTaskOperateSupport with Logging { + + private var id: Long = _ + + @Autowired + private var flowParsers: util.List[FlowEntranceJobParser] = _ + + @Autowired + private var flowDependencyResolvers: util.List[FlowDependencyResolver] = _ + + @Autowired + private var flowExecution: FlowExecution = _ + + private val SUBMIT_JOB_LOCK = "STATUS_CHANGED_LOCK".intern() + + protected def executeLine(code: String, storePath: String, alias: String): ExecuteResponse = { + val realCode = code.trim() + logger.info(s"flowExecutionEngine begins to run code:\n ${realCode.trim}") + + SuccessExecuteResponse() + } + + override def execute(executeRequest: ExecuteRequest): ExecuteResponse = { + executeRequest match { + case flowExecutionRequest: FlowExecutionRequest => + flowExecutionRequest.job match { + case job: FlowEntranceJob => + if (job.isCompleted) { + info(s"Flow(${job.getId}) isCompleted status is ${job.getState}") + return SuccessExecuteResponse() + } + Utils.tryCatch { + if (null == job.getFlow) { + Utils.tryCatch{ + for (flowParser <- flowParsers) { + flowParser.parse(job) + } + }{ t => + throw new FlowExecutionErrorException(90101, s"Failed to parser flow of job(${job.getId})", t) + } + } + this.SUBMIT_JOB_LOCK.synchronized { + for (flowDependencyResolver <- flowDependencyResolvers) { + flowDependencyResolver.resolvedFlow(job) + } + flowExecution.runJob(job) + } + } { t => + job.printLog(s"execute job failed: "+ExceptionUtils.getRootCauseMessage(t), "ERROR") + job.kill() + error(s"Failed to execute job: ${job.getId}", t) + } + } + } + SuccessExecuteResponse() + } + + override def kill(jobId: String): Boolean = { + true + } + + override def killAll(): Boolean = { + true + } + + override def pause(jobId: String): Boolean = { + true + } + + override def pauseAll(): Boolean = { + true + } + + override def resume(jobId: String): Boolean = { + true + } + + override def resumeAll(): Boolean = { + true + } + + override def close(): Unit = { + + } + + override def getId: Long = this.id + + override def state: ExecutorState = ExecutorState.Idle + + override def getExecutorInfo: ExecutorInfo = { + null + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowExecutionExecutorManagerImpl.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowExecutionExecutorManagerImpl.scala new file mode 100644 index 000000000..f2b6d8a7a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowExecutionExecutorManagerImpl.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.engine + +import java.util + +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.scheduler.executer.{Executor, ExecutorManager} +import org.apache.linkis.scheduler.listener.ExecutorListener +import org.apache.linkis.scheduler.queue.SchedulerEvent + +import scala.concurrent.duration.Duration + + +class FlowExecutionExecutorManagerImpl(flowEntranceEngine: FlowEntranceEngine) extends ExecutorManager with Logging{ + logger.info("FlowExecutionExecutorManagerImpl Registered") + private val idToEngines = new util.HashMap[Long, Executor] + + override def setExecutorListener(executorListener: ExecutorListener): Unit = { + + } + + + + protected override def createExecutor(schedulerEvent: SchedulerEvent): Executor = schedulerEvent match { + case job: FlowEntranceJob => + idToEngines.put(this.flowEntranceEngine.getId, this.flowEntranceEngine) + this.flowEntranceEngine + case _ => null + } + + override def askExecutor(schedulerEvent: SchedulerEvent): Option[Executor] = schedulerEvent match { + case job: FlowEntranceJob => + Some(createExecutor(schedulerEvent)) + } + + override def askExecutor(schedulerEvent: SchedulerEvent, wait: Duration): Option[Executor] = schedulerEvent match { + case job: FlowEntranceJob => + Some(createExecutor(schedulerEvent)) + } + + + override def getById(id: Long): Option[Executor] = { + Option(idToEngines.get(id)) + } + + override def getByGroup(groupName: String): Array[Executor] = { + null + } + + + + override def shutdown(): Unit = {} + + override def delete(executor: Executor): Unit = { + if (null != executor) { + executor.close() + idToEngines.remove(executor.getId) + }else { + logger.info("remove executor failed!") + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/entranceparser/FlowExecutionParser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/entranceparser/FlowExecutionParser.scala new file mode 100644 index 000000000..49875929d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/entranceparser/FlowExecutionParser.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.entranceparser + + +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.entrance.parser.CommonEntranceParser +import org.apache.linkis.entrance.persistence.PersistenceManager +import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.entity.task.RequestPersistTask +import org.apache.linkis.protocol.task.Task +import org.apache.linkis.scheduler.queue.Job +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +class FlowExecutionParser(persistenceManager: PersistenceManager) extends CommonEntranceParser(persistenceManager) with Logging { + + override def parseToJob(jobRequest: JobRequest): Job = { + val job = new FlowEntranceJob(persistenceManager) + job.setJobRequest(jobRequest) + job.setUser(jobRequest.getExecuteUser) + job.setCreator("FlowEntrance") + job.setParams(jobRequest.getParams.toMap[String, Any]) + job.setEntranceListenerBus(getEntranceContext.getOrCreateEventListenerBus) + job.setListenerEventBus(null) + job.setProgress(0f) + job.setEntranceContext(getEntranceContext) + job + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowExecutionErrorException.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowExecutionErrorException.scala new file mode 100644 index 000000000..36d4e8df9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowExecutionErrorException.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.exception + +import org.apache.linkis.common.exception.ErrorException + + +class FlowExecutionErrorException(errorCode: Int, desc: String) extends ErrorException(errorCode, desc){ + + def this(errorCode: Int, desc: String, t:Throwable){ + this(errorCode, desc) + initCause(t) + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowQueryErrorException.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowQueryErrorException.scala new file mode 100644 index 000000000..a6ceb4d11 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowQueryErrorException.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.exception + +import org.apache.linkis.common.exception.ErrorException + + +class FlowQueryErrorException(errorCode: Int, desc: String) extends ErrorException(errorCode, desc){ + + def this(errorCode: Int, desc: String, t:Throwable){ + this(errorCode, desc) + initCause(t) + } + +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala new file mode 100644 index 000000000..be502890b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala @@ -0,0 +1,89 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.execution + +import java.util.concurrent.{Executors, LinkedBlockingQueue, TimeUnit} + +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob +import com.webank.wedatasphere.dss.flow.execution.entrance.node.{NodeExecutionState, NodeRunner} +import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils +import org.apache.linkis.common.utils.Logging +import org.springframework.stereotype.Service + +import scala.collection.JavaConversions._ +import scala.collection.mutable.ArrayBuffer + + +@Service +class DefaultFlowExecution extends FlowExecution with Logging { + + + private val nodeRunnerQueue: LinkedBlockingQueue[NodeRunner] = new LinkedBlockingQueue[NodeRunner]() + + private val scheduledThreadPool = Executors.newScheduledThreadPool(FlowExecutionEntranceConfiguration.FLOW_EXECUTION_SCHEDULER_POOL_SIZE.getValue) + + private var pollerCount = 0 + + override def runJob(flowEntranceJob: FlowEntranceJob): Unit = { + + info(s"${flowEntranceJob.getId} Start to run executable node") + val scheduledNodes = flowEntranceJob.getFlowContext.getScheduledNodes + + if (!scheduledNodes.isEmpty) { + // get executableNodes + flowEntranceJob.getFlowContext synchronized { + val runners = new ArrayBuffer[NodeRunner]() + runners.addAll(scheduledNodes.values()) + val runningNodes = new ArrayBuffer[NodeRunner]() + val params = flowEntranceJob.getParams + runners.foreach { runner => + if (!FlowExecutionUtils.isSkippedNode(runner.getNode, params.asInstanceOf[java.util.Map[String, Any]])) { + info(s"scheduled node ${runner.getNode.getName} to running") + runner.fromScheduledTunToState(NodeExecutionState.Running) + // submit node runner + runningNodes.add(runner) + } + } + //避免skip节点导致任务重复提交,所以先要把所有running节点收集完成,改变状态,再来收集skipped节点,skipped节点被多次提交不受影响。 + runners.foreach { + runner => { + if (FlowExecutionUtils.isSkippedNode(runner.getNode, params.asInstanceOf[java.util.Map[String, Any]])) { + info(s"This node ${runner.getNode.getDSSNode.getName} Skipped in execution") + runner.fromScheduledTunToState(NodeExecutionState.Skipped) + } + } + } + info(s"${flowEntranceJob.getId} Submit nodes(${runningNodes.size}) to running") + runningNodes.foreach { node => + node.getNode.getDSSNode.getParams.get(FlowExecutionEntranceConfiguration.PROPS_MAP).asInstanceOf[java.util.Map[String, Any]].putAll(flowEntranceJob.getParams) + node.run() + nodeRunnerQueue.put(node) + if (pollerCount < FlowExecutionEntranceConfiguration.NODE_STATUS_POLLER_THREAD_SIZE.getValue) { + scheduledThreadPool.scheduleAtFixedRate(new NodeExecutionStatusPoller(nodeRunnerQueue), 1, + FlowExecutionEntranceConfiguration.NODE_STATUS_POLLER_SCHEDULER_TIME.getValue, TimeUnit.SECONDS) + pollerCount = pollerCount + 1 + } + } + } + } else { + info(s"${flowEntranceJob.getId} no executable nodes") + flowEntranceJob.tryCompleted + } + info(s"${flowEntranceJob.getId} finished to run node") + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/FlowExecution.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/FlowExecution.scala new file mode 100644 index 000000000..1779dd71e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/FlowExecution.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.execution + +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob + + +trait FlowExecution { + + def runJob(flowEntranceJob: FlowEntranceJob) + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/NodeExecutionStatusPoller.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/NodeExecutionStatusPoller.scala new file mode 100644 index 000000000..ff13d395a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/NodeExecutionStatusPoller.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.execution + + +import java.util.concurrent.LinkedBlockingQueue + +import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeRunner +import org.apache.linkis.common.utils.Logging + + +class NodeExecutionStatusPoller(nodeRunnerQueue: LinkedBlockingQueue[NodeRunner] ) extends Runnable with Logging{ + + + + override def run(): Unit = { + + val runner = nodeRunnerQueue.take() + + if(! runner.isLinkisJobCompleted) { + nodeRunnerQueue.put(runner) + } else { + info(s"node ${runner.getNode.getName} status is completed") + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala new file mode 100644 index 000000000..f97475fb2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala @@ -0,0 +1,177 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job + +import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException +import com.webank.wedatasphere.dss.flow.execution.entrance.listener.NodeRunnerListener +import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState +import com.webank.wedatasphere.dss.flow.execution.entrance.node.{NodeExecutionState, NodeRunner} +import com.webank.wedatasphere.dss.flow.execution.entrance.{FlowContext, FlowContextImpl} +import com.webank.wedatasphere.dss.workflow.core.entity.{Workflow, WorkflowNode} +import org.apache.linkis.common.log.LogUtils +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.entrance.execute.StorePathExecuteRequest +import org.apache.linkis.entrance.job.EntranceExecutionJob +import org.apache.linkis.entrance.persistence.PersistenceManager +import org.apache.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteRequest, SuccessExecuteResponse} +import org.apache.linkis.scheduler.queue.SchedulerEventState.Running +import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} + +import scala.collection.JavaConversions._ +import scala.collection.mutable.ArrayBuffer + +class FlowEntranceJob(persistManager:PersistenceManager) extends EntranceExecutionJob(persistManager) with NodeRunnerListener { + + private var flow: Workflow = _ + + private val flowContext: FlowContext = new FlowContextImpl + + +// @BeanProperty var dwsProject: DWSProject = _ + + def setFlow(flow: Workflow): Unit = this.flow = flow + + def getFlow: Workflow = this.flow + + def getFlowContext: FlowContext = this.flowContext + + private val STATUS_CHANGED_LOCK = "STATUS_CHANGED_LOCK".intern() + + override def init():Unit = { + + } + + override def jobToExecuteRequest(): ExecuteRequest = { + new ExecuteRequest with StorePathExecuteRequest with FlowExecutionRequest { + override val code: String = FlowEntranceJob.this.getJobRequest.getExecutionCode + override val storePath: String = FlowEntranceJob.this.getJobRequest match { +// case requestPersistTask: RequestPersistTask => requestPersistTask.getResultLocation + case _ => "" + } + override val job: Job = FlowEntranceJob.this + } + } + + override def run(): Unit = { + setResultSize(0) + if (!isScheduled) return + startTime = System.currentTimeMillis + Utils.tryAndWarn(transition(Running)) + getExecutor.execute(jobToExecuteRequest()) + } + + + override def onStatusChanged(fromState: NodeExecutionState, toState: NodeExecutionState, node: WorkflowNode): Unit = { + + val nodeName = node.getDSSNode.getName + toState match { + case NodeExecutionState.Failed => + printLog(s"Failed to execute node($nodeName),prepare to kill flow job", "ERROR") + if(NodeExecutionState.isRunning(fromState)) + FlowContext.changedNodeState(this.getFlowContext.getRunningNodes, this.getFlowContext.getFailedNodes, node, "node execute fail") + this.kill() + info(s"Succeed to kill flow job") + case NodeExecutionState.Cancelled => + printLog(s"node($nodeName) has cancelled execution", "WARN") + if(NodeExecutionState.isRunning(fromState)) + FlowContext.changedNodeState(this.getFlowContext.getRunningNodes, this.getFlowContext.getFailedNodes, node, "node has cancelled") + if(NodeExecutionState.isScheduled(fromState)) + FlowContext.changedNodeState(this.getFlowContext.getScheduledNodes, this.getFlowContext.getFailedNodes, node, "node has cancelled") + case NodeExecutionState.Skipped => + printLog(s"node($nodeName) has skipped execution from $fromState", "WARN") + FlowContext.changedNodeState(this.getFlowContext.getScheduledNodes, this.getFlowContext.getSkippedNodes, node, "node has skipped") + //Trigger the next execution + this.STATUS_CHANGED_LOCK.synchronized { + getExecutor.execute(jobToExecuteRequest()) + } + case NodeExecutionState.Succeed => + printLog(s"Succeed to execute node($nodeName)", "INFO") + if (NodeExecutionState.isRunning(fromState)) + FlowContext.changedNodeState(this.getFlowContext.getRunningNodes, this.getFlowContext.getSucceedNodes, node, "node execute success") + //Trigger the next execution + this.STATUS_CHANGED_LOCK.synchronized { + getExecutor.execute(jobToExecuteRequest()) + } + case NodeExecutionState.Running => + printLog(s"Start to execute node($nodeName) ", "INFO") + if(NodeExecutionState.isScheduled(fromState)) + FlowContext.changedNodeState(this.getFlowContext.getScheduledNodes, this.getFlowContext.getRunningNodes, node, "node in running") + + case NodeExecutionState.Scheduled => + printLog(s"node($nodeName) from inited to scheduled", "INFO") + if(NodeExecutionState.isInited(fromState)) + FlowContext.changedNodeState(this.getFlowContext.getPendingNodes, this.getFlowContext.getScheduledNodes, node, "node in scheduled") + case _ => + } + tryCompleted + } + + def printLog(log:String, level:String): Unit = level match { + case "INFO" => + info(log) + getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateInfo(log))) + case "WARN" => + warn(log) + getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateWarn(log))) + case "ERROR" => + error(log) + getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateERROR(log))) + case _ => + } + + override def kill(): Unit = if (! SchedulerEventState.isCompleted(this.getState)) this synchronized { + if(! SchedulerEventState.isCompleted(this.getState)){ + super.kill() + Utils.tryAndWarn(this.killNodes) + Utils.tryAndWarn(transitionCompleted(ErrorExecuteResponse(s"execute job(${getId}) failed!", new FlowExecutionErrorException(90101, s"This Flow killed by user")))) + } + } + + override def cancel(): Unit = if (! SchedulerEventState.isCompleted(this.getState)) this synchronized { + if(! SchedulerEventState.isCompleted(this.getState)){ + Utils.tryAndWarn(this.killNodes) + super.cancel() + Utils.tryAndWarn(transitionCompleted(ErrorExecuteResponse(s"cancel job(${getId}) execution!", new FlowExecutionErrorException(90101, s"This Flow killed by user")))) + } + } + + def isFlowCompleted: Boolean = this.getFlowContext.getRunningNodes.isEmpty && this.getFlowContext.getPendingNodes.isEmpty && this.getFlowContext.getScheduledNodes.isEmpty + + + def tryCompleted: Unit = { + if (this.isFlowCompleted) { + info(s"This Flow(${getId}) is Completed") + /*transition(SchedulerEventState.Succeed)*/ + if(! SchedulerEventState.isCompleted(this.getState)) + transitionCompleted(SuccessExecuteResponse()) + } + } + + def killNodes: Unit = { + val runners = new ArrayBuffer[NodeRunner]() + runners.addAll(this.getFlowContext.getRunningNodes.values()) + for (node <- runners) { + Utils.tryAndWarn(node.cancel()) + } + + + } + + + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionRequest.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionRequest.scala new file mode 100644 index 000000000..79c5bb0e2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionRequest.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job + +import org.apache.linkis.scheduler.queue.Job + + +trait FlowExecutionRequest { + val job:Job +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowEntranceJobParser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowEntranceJobParser.scala new file mode 100644 index 000000000..8f05dbe35 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowEntranceJobParser.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job.parser + +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob + + +trait FlowEntranceJobParser { + + def parse(flowEntranceJob: FlowEntranceJob):Unit + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobFlowParser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobFlowParser.scala new file mode 100644 index 000000000..404117bb9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobFlowParser.scala @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job.parser +import com.webank.wedatasphere.dss.common.protocol.{ProxyUserCheckRequest, RequestQueryWorkFlow, ResponseProxyUserCheck} +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob +import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseQueryWorkflow +import com.webank.wedatasphere.dss.workflow.core.WorkflowFactory +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.rpc.Sender +import org.apache.commons.lang3.StringUtils +import org.springframework.core.annotation.Order +import org.springframework.stereotype.Component + + +@Order(1) +@Component +class FlowJobFlowParser extends FlowEntranceJobParser with Logging { + + override def parse(flowEntranceJob: FlowEntranceJob): Unit = { + info(s"Start to parse flow of Job(${flowEntranceJob.getId}).") + val code = flowEntranceJob.jobToExecuteRequest().code + val label = flowEntranceJob.getParams.getOrDefault(DSSCommonUtils.DSS_LABELS_KEY, DSSCommonUtils.ENV_LABEL_VALUE_DEV).asInstanceOf[String] + val codeLanguageLabel = FlowExecutionUtils.getRunTypeLabel(flowEntranceJob.getJobRequest().getLabels) + if("json" == codeLanguageLabel.getCodeType // 兼容老版本 + || "flow" == codeLanguageLabel.getCodeType) { + val flowExecutionCode =DSSCommonUtils.COMMON_GSON.fromJson(code, classOf[java.util.Map[String, Any]]) + val flowId = flowExecutionCode.get("flowId").asInstanceOf[Number].longValue() + val workflow = getDSSScheduleFlowById(flowEntranceJob.getUser, flowId, label) + flowEntranceJob.setFlow(workflow) + info(s"Finished to parse flow of Job(${flowEntranceJob.getId}).") + } + } + + private def getDSSScheduleFlowById(userName: String , flowId:Long, label:String): Workflow = { + val req = RequestQueryWorkFlow(userName, flowId) + val sendWorkflowName = FlowExecutionEntranceConfiguration.WORKFLOW_APPLICATION_NAME.getValue + label.toUpperCase + logger.info("Send query workflow json to "+ sendWorkflowName) + val response: ResponseQueryWorkflow= Sender.getSender(sendWorkflowName) + .ask(req).asInstanceOf[ResponseQueryWorkflow] + WorkflowFactory.INSTANCE.getJsonToFlowParser.parse(response.getDssFlow) + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala new file mode 100644 index 000000000..81671406e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala @@ -0,0 +1,101 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job.parser + +import java.util +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration._ +import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob +import com.webank.wedatasphere.dss.flow.execution.entrance.node.DefaultNodeRunner +import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowWithContextImpl +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.entrance.execute.EntranceJob +import org.springframework.core.annotation.Order +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +@Order(10) +@Component +class FlowJobNodeParser extends FlowEntranceJobParser with Logging{ + + override def parse(flowEntranceJob: FlowEntranceJob): Unit = { + info(s"${flowEntranceJob.getId} Start to parse node of flow.") + val flow = flowEntranceJob.getFlow + val flowContext = flowEntranceJob.getFlowContext + if(null == flow) throw new FlowExecutionErrorException(90101, "This fow of job is empty ") + val nodes = flow.getWorkflowNodes + for (node <- nodes) { + + val nodeName = node.getName + val propsMap = new util.HashMap[String, String]() + val proxyUser = if (node.getDSSNode.getUserProxy == null) flowEntranceJob.getUser else node.getDSSNode.getUserProxy + propsMap.put(FLOW_NAME, flow.getName) + propsMap.put(JOB_ID, nodeName) + val projectName =flowEntranceJob.asInstanceOf[EntranceJob].getJobRequest.getSource.get("projectName") + propsMap.put(PROJECT_NAME,projectName) + propsMap.put(FLOW_SUBMIT_USER, flowEntranceJob.getUser) + info(s"Job(${flowEntranceJob.getId}) with nodeName: $nodeName and node: ${node.getNodeType}.") + propsMap.put(LinkisJobExecutionConfiguration.LINKIS_TYPE, node.getNodeType) + + propsMap.put(PROXY_USER, proxyUser) + propsMap.put(COMMAND, LinkisJobExecutionUtils.gson1.toJson(node.getDSSNode.getJobContent)) + + var params = node.getDSSNode.getParams + if (params == null) { + params = new util.HashMap[String,AnyRef]() + node.getDSSNode.setParams(params) + } + val flowVar = new util.HashMap[String, AnyRef]() + val properties = flow.getFlowProperties + if(properties != null) { + for(proper <- properties){ + flowVar.putAll(proper) + } + } + + propsMap.put(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID, flowEntranceJob.getId) + + flow match { + case workflow: WorkflowWithContextImpl => + propsMap.put(CONTEXT_ID, workflow.getContextID) + case _ => + } + + params.put(PROPS_MAP, propsMap) + params.put(FLOW_VAR_MAP, flowVar) + val flowNameAndResources = new util.HashMap[String, util.List[BMLResource]]() + flowNameAndResources.put("flow." + flow.getName + LinkisJobExecutionConfiguration.RESOURCES_NAME, FlowExecutionUtils.resourcesAdaptation(flow.getFlowResources)) + params.put(FLOW_RESOURCES, flowNameAndResources) + + val pendingNodeMap = flowContext.getPendingNodes + + val nodeRunner = pendingNodeMap.getOrDefault(nodeName, new DefaultNodeRunner) + nodeRunner.setNodeRunnerListener(flowEntranceJob) + nodeRunner.setNode(node) + pendingNodeMap.put(nodeName, nodeRunner) + } + info(s"${flowEntranceJob.getId} finished to parse node of flow") + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJsonFlowParser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJsonFlowParser.scala new file mode 100644 index 000000000..2c470c267 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJsonFlowParser.scala @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job.parser +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob +import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow +import com.webank.wedatasphere.dss.workflow.core.WorkflowFactory +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.governance.common.entity.job.JobRequest +import org.springframework.core.annotation.Order + +import scala.collection.convert.wrapAsScala._ + + +@Order(2) +class FlowJsonFlowParser extends FlowEntranceJobParser with Logging { + + override def parse(flowEntranceJob: FlowEntranceJob): Unit = { + info(s"Start to parse flow of Job(${flowEntranceJob.getId}).") + val codeLanguageLabel = FlowExecutionUtils.getRunTypeLabel(flowEntranceJob.getJobRequest().getLabels) + if("flowjson" == codeLanguageLabel.getCodeType.toLowerCase) { + val workflow = WorkflowFactory.INSTANCE.getJsonToFlowParser.parse(createDSSFlow(flowEntranceJob.getJobRequest())) + flowEntranceJob.setFlow(workflow) + info(s"Finished to parse flow of Job(${flowEntranceJob.getId}).") + } + } + + private def createDSSFlow(jobRequest: JobRequest): DSSFlow = { + val dssFlow = new DSSFlow + dssFlow.setFlowJson(jobRequest.getExecutionCode) + dssFlow.setCreator(jobRequest.getExecuteUser) + val name = if(jobRequest.getSource != null) jobRequest.getSource.values().mkString("-") + else jobRequest.getSubmitUser + "-Flow" + dssFlow.setName(name) + dssFlow + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/listener/NodeRunnerListener.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/listener/NodeRunnerListener.scala new file mode 100644 index 000000000..760cf3740 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/listener/NodeRunnerListener.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.listener + +import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode + + +trait NodeRunnerListener { + + def onStatusChanged(fromState: NodeExecutionState, toState: NodeExecutionState, node: WorkflowNode): Unit +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/log/FlowExecutionLog.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/log/FlowExecutionLog.scala new file mode 100644 index 000000000..12978822f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/log/FlowExecutionLog.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.log + + +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog +import org.apache.linkis.common.utils.Logging + + +class FlowExecutionLog(log:Logging) extends LinkisJobExecutionLog { + override def info(message: scala.Any, t: Throwable): Unit = { + if(message != null && FlowExecutionEntranceConfiguration.NODE_PRINT_FLAG.getValue) log.info(message.toString, t) + } + + override def warn(message: scala.Any, t: Throwable): Unit = { + if(message != null && FlowExecutionEntranceConfiguration.NODE_PRINT_FLAG.getValue) log.warn(message.toString, t) + } + + override def error(message: scala.Any, t: Throwable): Unit = { + if(message != null && FlowExecutionEntranceConfiguration.NODE_PRINT_FLAG.getValue) log.error(message.toString, t) + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppConnJobBuilder.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppConnJobBuilder.scala new file mode 100644 index 000000000..23f8e00fa --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppConnJobBuilder.scala @@ -0,0 +1,116 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.node + +import java.util + +import com.webank.wedatasphere.dss.common.entity.Resource +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.flow.execution.entrance.job._ +import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource +import com.webank.wedatasphere.dss.linkis.node.execution.job._ +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode +import org.apache.commons.lang.StringUtils + + +object AppConnJobBuilder { + + def builder():FlowBuilder = new FlowBuilder + + class FlowBuilder extends Builder { + + private var jobProps: java.util.Map[String, String] = _ + + private var node: WorkflowNode = _ + + def setJobProps(jobProps: java.util.Map[String, String]): FlowBuilder = { + this.jobProps = jobProps + this + } + + def setNode(node: WorkflowNode): FlowBuilder = { + this.node = node + this + } + + override protected def getJobType: String = jobProps.getOrDefault(LinkisJobExecutionConfiguration.LINKIS_TYPE, LinkisJobExecutionConfiguration.LINKIS_DEFAULT_TYPE.getValue(jobProps)) + + override protected def creatLinkisJob(isLinkisType: Boolean): LinkisJob = { + if(isLinkisType){ + val linkisJob = new FlowExecutionCommonLinkisJob + linkisJob.setJobProps(this.jobProps) + linkisJob + } else { + val linkisJob = new FlowExecutionAppConnLinkisJob + linkisJob.setJobProps(this.jobProps) + linkisJob + } + } + + override protected def fillJobInfo(job: Job): Unit = { + if(StringUtils.isNotEmpty(jobProps.get(FlowExecutionEntranceConfiguration.COMMAND))) + job.setCode(jobProps.get(FlowExecutionEntranceConfiguration.COMMAND)) + job.setParams(new util.HashMap[String,AnyRef]()) + val runtimeMap = new util.HashMap[String, AnyRef]() + runtimeMap.put(FlowExecutionEntranceConfiguration.NODE_NAME, jobProps.get(FlowExecutionEntranceConfiguration.JOB_ID)) + runtimeMap.put(FlowExecutionEntranceConfiguration.WORKSPACE, jobProps.get(FlowExecutionEntranceConfiguration.WORKSPACE)) + runtimeMap.put(DSSCommonUtils.DSS_LABELS_KEY, jobProps.get(DSSCommonUtils.DSS_LABELS_KEY)) + job.setRuntimeParams(runtimeMap) + } + + override protected def fillLinkisJobInfo(linkisJob: LinkisJob): Unit = { + this.node.getDSSNode.getParams.get(FlowExecutionEntranceConfiguration.NODE_CONFIGURATION_KEY) match { + case configuration:util.Map[String, AnyRef] => + linkisJob.setConfiguration(configuration) + case _ => + } + this.node.getDSSNode.getParams.remove(FlowExecutionEntranceConfiguration.FLOW_VAR_MAP) match { + case flowVar:util.Map[String, AnyRef] => + linkisJob.setVariables(flowVar) + case _ => + } + linkisJob.setSource(getSource) + } + + override protected def fillCommonLinkisJobInfo(linkisAppConnJob: CommonLinkisJob): Unit = { + linkisAppConnJob.setJobResourceList(FlowExecutionUtils.resourcesAdaptation(this.node.getDSSNode.getResources)) + this.node.getDSSNode.getParams.remove(FlowExecutionEntranceConfiguration.PROJECT_RESOURCES) match { + case projectResources:util.List[Resource] => + linkisAppConnJob.setProjectResourceList(FlowExecutionUtils.resourcesAdaptation(projectResources)) + case _ => + } + this.node.getDSSNode.getParams.remove(FlowExecutionEntranceConfiguration.FLOW_RESOURCES) match { + case flowResources:util.HashMap[String, util.List[BMLResource]] => + linkisAppConnJob.setFlowNameAndResources(flowResources) + case _ => + } + } + + private def getSource: util.Map[String, String] = { + val source = new util.HashMap[String, String]() + source.put("projectName", jobProps.get(FlowExecutionEntranceConfiguration.PROJECT_NAME)) + source.put("flowName", jobProps.get(FlowExecutionEntranceConfiguration.FLOW_NAME)) + source.put("nodeName", jobProps.get(FlowExecutionEntranceConfiguration.JOB_ID)) + source + } + + override protected def getContextID(job: Job): String = jobProps.get(FlowExecutionEntranceConfiguration.CONTEXT_ID) + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala new file mode 100644 index 000000000..b9a17d3a4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala @@ -0,0 +1,156 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.node + +import java.util + +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.flow.execution.entrance.listener.NodeRunnerListener +import com.webank.wedatasphere.dss.flow.execution.entrance.log.FlowExecutionLog +import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration +import com.webank.wedatasphere.dss.linkis.node.execution.execution.impl.LinkisNodeExecutionImpl +import com.webank.wedatasphere.dss.linkis.node.execution.job.{JobTypeEnum, LinkisJob} +import com.webank.wedatasphere.dss.linkis.node.execution.listener.LinkisExecutionListener +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode +import org.apache.linkis.common.exception.ErrorException +import org.apache.linkis.common.utils.{Logging, Utils} + + +class DefaultNodeRunner extends NodeRunner with Logging { + + private var node: WorkflowNode = _ + + private var linkisJob: LinkisJob = _ + + private var canceled: Boolean = false + + private var status: NodeExecutionState = NodeExecutionState.Inited + + private var nodeRunnerListener: NodeRunnerListener = _ + + private var executedInfo: String = _ + + private var startTime: Long = _ + + private var nowTime:Long = _ + + private var lastGetStatusTime: Long = 0 + + override def getNode: WorkflowNode = this.node + + def setNode(schedulerNode: WorkflowNode): Unit = { + this.node = schedulerNode + } + + override def getLinkisJob: LinkisJob = this.linkisJob + + + override def isCanceled: Boolean = this.canceled + + override def getStatus: NodeExecutionState = { + this.status + } + + override def isLinkisJobCompleted: Boolean = Utils.tryCatch{ + + val interval = System.currentTimeMillis() - lastGetStatusTime + + if ( interval < FlowExecutionEntranceConfiguration.NODE_STATUS_INTERVAL.getValue){ + return false + } + + lastGetStatusTime = System.currentTimeMillis() + + if(NodeExecutionState.isCompleted(getStatus)) return true + val toState = NodeExecutionState.withName(LinkisNodeExecutionImpl.getLinkisNodeExecution.getState(this.linkisJob)) + if (NodeExecutionState.isCompleted(toState)) { + val listener = LinkisNodeExecutionImpl.getLinkisNodeExecution.asInstanceOf[LinkisExecutionListener] + listener.onStatusChanged(getStatus.toString, toState.toString, this.linkisJob) + this.transitionState(toState) + info(s"Finished to execute node of ${this.node.getName}") + true + } else { + false + } + }{ + case e:ErrorException => logger.warn(s"failed to get ${this.node.getName} state", e) + false + case t :Throwable => logger.error(s"failed to get ${this.node.getName} state", t) + true + } + + override def setNodeRunnerListener(nodeRunnerListener: NodeRunnerListener): Unit = this.nodeRunnerListener = nodeRunnerListener + + override def getNodeRunnerListener: NodeRunnerListener = this.nodeRunnerListener + + override def run(): Unit = { + info(s"start to run node of ${node.getName}") + try { + val jobProps = node.getDSSNode.getParams.remove(FlowExecutionEntranceConfiguration.PROPS_MAP) match { + case propsMap: util.Map[String, String] => propsMap + case _ => new util.HashMap[String, String]() + } + this.linkisJob = AppConnJobBuilder.builder().setNode(node).setJobProps(jobProps).build.asInstanceOf[LinkisJob] + this.linkisJob.setLogObj(new FlowExecutionLog(this)) + //set start time + this.setStartTime(System.currentTimeMillis()) + if (JobTypeEnum.EmptyJob == this.linkisJob.getJobType) { + warn("This node is empty type") + this.transitionState(NodeExecutionState.Succeed) + return + } + this.linkisJob.getJobProps.put(LinkisJobExecutionConfiguration.LINKIS_VERSION_KEY, LinkisJobExecutionConfiguration.LINKIS_DEFAULT_VERSION.getValue) + LinkisNodeExecutionImpl.getLinkisNodeExecution.runJob(this.linkisJob) + info(s"Finished to run node of ${node.getName}") + } catch { + case t: Throwable => + warn(s"Failed to execute node of ${node.getName}", t) + this.transitionState(NodeExecutionState.Failed) + + } + + } + + + override def cancel(): Unit = if (!this.canceled) this synchronized { + if (this.canceled) return + this.canceled = true + if (this.linkisJob != null && this.linkisJob.getJobExecuteResult != null) + LinkisNodeExecutionImpl.getLinkisNodeExecution.cancel(this.linkisJob) + this.transitionState(NodeExecutionState.Cancelled) + warn(s"This node(${node.getName}) has been canceled") + } + + override def pause(): Unit = {} + + override def resume(): Boolean = true + + override def setStatus(nodeExecutionState: NodeExecutionState): Unit = this.status = nodeExecutionState + + override def getNodeExecutedInfo(): String = this.executedInfo + + override def setNodeExecutedInfo(info: String): Unit = this.executedInfo = info + + override def getStartTime(): Long = this.startTime + + override def setStartTime(startTime: Long): Unit = this.startTime = startTime + + override def getNowTime(): Long = this.nowTime + + override def setNowTime(nowTime: Long): Unit = this.nowTime = nowTime +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeExecutionState.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeExecutionState.scala new file mode 100644 index 000000000..d44b7dfc4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeExecutionState.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.node + + +object NodeExecutionState extends Enumeration { + + type NodeExecutionState = Value + + val Inited, WaitForRetry, Scheduled, Running, Succeed, Failed, Cancelled, Skipped ,Timeout = Value + + def isRunning(jobState: NodeExecutionState) = Running == jobState + + def isScheduled(jobState: NodeExecutionState) = Scheduled == jobState + + def isInited(jobState: NodeExecutionState) = Inited == jobState + + def isCompleted(jobState: NodeExecutionState) = jobState match { + case Inited | Scheduled | Running | WaitForRetry => false + case _ => true + } + + def isSucceed(jobState: NodeExecutionState) = Succeed == jobState + + def isFailed(jobState: NodeExecutionState):Boolean = Failed == jobState +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeRunner.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeRunner.scala new file mode 100644 index 000000000..3fefe9077 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeRunner.scala @@ -0,0 +1,102 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.node + +import java.util.concurrent.Future + +import com.webank.wedatasphere.dss.flow.execution.entrance.FlowContext +import com.webank.wedatasphere.dss.flow.execution.entrance.listener.NodeRunnerListener +import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState +import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode +import org.apache.linkis.common.utils.{Logging, Utils} + + +abstract class NodeRunner extends Runnable with Logging{ + + private[flow] var future: Future[_] = _ + + private var flowContext:FlowContext = _ + + + + def getFlowContext:FlowContext = this.flowContext + + def setFlowContext(flowContext: FlowContext): Unit = { + this.flowContext = flowContext + } + + def getNode: WorkflowNode + + def setNode(node: WorkflowNode):Unit + + def getLinkisJob: LinkisJob + + def cancel(): Unit + + def pause(): Unit + + def resume(): Boolean + + def isCanceled: Boolean + + def getStatus: NodeExecutionState + + def isLinkisJobCompleted: Boolean + + def setStatus(nodeExecutionState: NodeExecutionState): Unit + + def setNodeRunnerListener(nodeRunnerListener: NodeRunnerListener): Unit + + def getNodeRunnerListener: NodeRunnerListener + + def getNodeExecutedInfo(): String + + def setNodeExecutedInfo(info: String ):Unit + + def getStartTime(): Long + + def setStartTime(startTime: Long): Unit + + def getNowTime():Long + + def setNowTime(nowTime: Long):Unit + + protected def transitionState(toState: NodeExecutionState): Unit = Utils.tryAndWarn{ + if (getStatus == toState) return + info(s"from state $getStatus to $toState") + this.getNodeRunnerListener.onStatusChanged(getStatus, toState, this.getNode) + this.setStatus(toState) + } + + + def tunToScheduled(): Boolean = if (! NodeExecutionState.isInited(this.getStatus)) false else this synchronized { + if (! NodeExecutionState.isInited(this.getStatus)) false else { + transitionState(NodeExecutionState.Scheduled) + true + } + } + + def fromScheduledTunToState(state: NodeExecutionState): Boolean = if (! NodeExecutionState.isScheduled(this.getStatus) ) false else this synchronized { + if (! NodeExecutionState.isScheduled(this.getStatus)) false else { + transitionState(state) + true + } + } + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolver.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolver.scala new file mode 100644 index 000000000..0f8b898db --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolver.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.resolver + +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob + + +trait FlowDependencyResolver { + def resolvedFlow(flowJob:FlowEntranceJob) +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolverImpl.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolverImpl.scala new file mode 100644 index 000000000..f3aca4919 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolverImpl.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.resolver + +import java.util + +import com.webank.wedatasphere.dss.flow.execution.entrance.FlowContext +import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob +import org.apache.linkis.common.utils.Logging +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +@Component +class FlowDependencyResolverImpl extends FlowDependencyResolver with Logging { + override def resolvedFlow(flowJob: FlowEntranceJob) = { + + info(s"${flowJob.getId} Start to get executable node") + + val flowContext: FlowContext = flowJob.getFlowContext + val nodes = flowContext.getPendingNodes.toMap.values.map(_.getNode) + + def isAllParentDependencyCompleted(parents:util.List[String]): Boolean = { + for (parent <- parents){ + if( ! flowContext.isNodeCompleted(parent)) return false + } + true + } + nodes.foreach{ node => + val nodeName = node.getName + def isCanExecutable: Boolean = { + (flowContext.getPendingNodes.containsKey(nodeName) + && !FlowContext.isNodeRunning(nodeName, flowContext) + && !flowContext.isNodeCompleted(nodeName) + && isAllParentDependencyCompleted(node.getDependencys)) + } + if (isCanExecutable) flowContext synchronized { + if (isCanExecutable) flowContext.getPendingNodes.get(nodeName).tunToScheduled() + } + } + info(s"${flowJob.getId} Finished to get executable node(${flowContext.getScheduledNodes.size()})") + + } + + +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/NodeSkipStrategy.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/NodeSkipStrategy.scala new file mode 100644 index 000000000..5c82709b8 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/NodeSkipStrategy.scala @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.strategy + +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode + +trait NodeSkipStrategy { + /** + * 判断节点是否需要跳过 + * + * @param node 节点信息 + * @param paramsMap 执行入参 + * @param isReversedChoose 是否是反选 + * @return true/false + * + * + */ + def isSkippedNode(node: WorkflowNode, paramsMap: java.util.Map[String, Any], isReversedChoose: Boolean): Boolean = { + if (FlowExecutionEntranceConfiguration.SKIP_NODES.getValue.split(",").exists(_.equalsIgnoreCase(node.getNodeType))) { + return true + } + val nodeIdObj = paramsMap.get("nodeID") + if (nodeIdObj != null) { + if (isReversedChoose) { + return !nodeIdObj.toString.split(",").exists(_.equalsIgnoreCase(node.getId)) + } else { + return nodeIdObj.toString.split(",").exists(_.equalsIgnoreCase(node.getId)) + } + } + false + } +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/StrategyFactory.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/StrategyFactory.scala new file mode 100644 index 000000000..4992cc270 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/StrategyFactory.scala @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.strategy + +import com.webank.wedatasphere.dss.flow.execution.entrance.enums.ExecuteStrategyEnum +import com.webank.wedatasphere.dss.flow.execution.entrance.strategy.impl.{ExecuteNodeSkipStrategy, ReExecuteNodeSkipStrategy, SelectedExecuteNodeSkipStrategy} + +object StrategyFactory { + + def getNodeSkipStrategy(executeStrategy: String): NodeSkipStrategy = { + val skipStrategy = ExecuteStrategyEnum.getEnum(executeStrategy) match { + case ExecuteStrategyEnum.IS_RE_EXECUTE => new ReExecuteNodeSkipStrategy + case ExecuteStrategyEnum.IS_SELECTED_EXECUTE => new SelectedExecuteNodeSkipStrategy + case ExecuteStrategyEnum.IS_EXECUTE => new ExecuteNodeSkipStrategy + case _ => new ExecuteNodeSkipStrategy + } + skipStrategy + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/ExecuteNodeSkipStrategy.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/ExecuteNodeSkipStrategy.scala new file mode 100644 index 000000000..dddca762e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/ExecuteNodeSkipStrategy.scala @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.strategy.impl + +import java.util + +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.flow.execution.entrance.strategy.NodeSkipStrategy +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode + +class ExecuteNodeSkipStrategy extends NodeSkipStrategy { + override def isSkippedNode(node: WorkflowNode, paramsMap: util.Map[String, Any], isReversedChoose: Boolean): Boolean = { + FlowExecutionEntranceConfiguration.SKIP_NODES.getValue.split(",").exists(_.equalsIgnoreCase(node.getNodeType)) + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/ReExecuteNodeSkipStrategy.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/ReExecuteNodeSkipStrategy.scala new file mode 100644 index 000000000..3b3b2a6ca --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/ReExecuteNodeSkipStrategy.scala @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.strategy.impl + + + +import com.webank.wedatasphere.dss.flow.execution.entrance.strategy.NodeSkipStrategy + +class ReExecuteNodeSkipStrategy extends NodeSkipStrategy { +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/SelectedExecuteNodeSkipStrategy.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/SelectedExecuteNodeSkipStrategy.scala new file mode 100644 index 000000000..17ddcd55b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/strategy/impl/SelectedExecuteNodeSkipStrategy.scala @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.flow.execution.entrance.strategy.impl + + +import java.util + +import com.webank.wedatasphere.dss.flow.execution.entrance.strategy.NodeSkipStrategy +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode + +class SelectedExecuteNodeSkipStrategy extends NodeSkipStrategy { + override def isSkippedNode(node: WorkflowNode, paramsMap: util.Map[String, Any], isReversedChoose: Boolean): Boolean = { + super.isSkippedNode(node, paramsMap, isReversedChoose = true) + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala new file mode 100644 index 000000000..00a3d65a7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala @@ -0,0 +1,117 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.utils + +import java.lang.reflect.Type +import java.util.Date +import java.{lang, util} + +import com.google.gson._ +import com.webank.wedatasphere.dss.common.entity.Resource +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration +import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException +import com.webank.wedatasphere.dss.flow.execution.entrance.strategy.StrategyFactory +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode +import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.entity.task.RequestPersistTask +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel +import org.apache.linkis.manager.label.utils.LabelUtil + +import scala.collection.JavaConversions._ +import scala.collection.JavaConverters.asScalaBufferConverter + +object FlowExecutionUtils { + + def isSkippedNode(node: WorkflowNode, paramsMap: java.util.Map[String, Any]): Boolean = { + val executeStrategy = paramsMap.get("executeStrategy") + if (executeStrategy != null) { + return StrategyFactory.getNodeSkipStrategy(executeStrategy.toString).isSkippedNode(node, paramsMap, isReversedChoose = false) + } + false + } + + def getRunTypeLabel(labels: util.List[Label[_]]): CodeLanguageLabel = { + labels.find { + case label: CodeLanguageLabel => true + case _ => false + }.map(_.asInstanceOf[CodeLanguageLabel]).getOrElse(throw new FlowExecutionErrorException(90106, "Cannot find runType label.")) + } + + def isSignalNode(jobType: String): Boolean = { + FlowExecutionEntranceConfiguration.SIGNAL_NODES.getValue.split(",").exists(_.equalsIgnoreCase(jobType)) + } + + def isAppConnJob(engineType: String): Boolean = LinkisJobExecutionConfiguration.APPCONN.equalsIgnoreCase(engineType) + + + def resourcesAdaptation(resources: util.List[Resource]): util.ArrayList[BMLResource] = { + val bmlResources = new util.ArrayList[BMLResource]() + for (resource <- resources) { + bmlResources.add(resourceConvertToBMLResource(resource)) + } + bmlResources + } + + def resourceConvertToBMLResource(resource: Resource): BMLResource = { + val bmlResource = new BMLResource() + bmlResource.setFileName(resource.getFileName) + bmlResource.setResourceId(resource.getResourceId) + bmlResource.setVersion(resource.getVersion) + bmlResource + } + + + def jobRequest2RequestPersistTask(jobReq: JobRequest): RequestPersistTask = { + if (null == jobReq) return null + val persistTask = new RequestPersistTask + persistTask.setTaskID(jobReq.getId) + persistTask.setExecId(jobReq.getReqId) + // jobHistory.setPriority(jobReq.getProgress) + persistTask.setSubmitUser(jobReq.getSubmitUser) + persistTask.setUmUser(jobReq.getExecuteUser) + persistTask.setSource(jobReq.getSource) + // persistTask.setExecutionCode(jobReq.getExecutionCode) + if (null != jobReq.getLabels) { + val labelMap = new util.HashMap[String, String](jobReq.getLabels.size()) + jobReq.getLabels.asScala.map(l => l.getLabelKey -> l.getStringValue).foreach(kv => labelMap.put(kv._1, kv._2)) + persistTask.setLabels(jobReq.getLabels) + } + if (null != jobReq.getParams) persistTask.setParams(jobReq.getParams) + persistTask.setProgress(jobReq.getProgress.toFloat) + persistTask.setStatus(jobReq.getStatus) + persistTask.setLogPath(jobReq.getLogPath) + persistTask.setErrCode(jobReq.getErrorCode) + persistTask.setErrDesc(jobReq.getErrorDesc) + if (null != jobReq.getCreatedTime) persistTask.setCreatedTime(new Date(jobReq.getCreatedTime.getTime)) + if (null != jobReq.getUpdatedTime) persistTask.setUpdatedTime(new Date(jobReq.getUpdatedTime.getTime)) + persistTask.setInstance(jobReq.getInstances) + // if (null != jobReq.getMetrics) persistTask.set(gson.toJson(jobReq.getMetrics)) + val engineType = LabelUtil.getEngineType(jobReq.getLabels) + persistTask.setEngineType(engineType) + persistTask + } + + implicit val gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls + .registerTypeAdapter(classOf[java.lang.Double], new JsonSerializer[java.lang.Double] { + override def serialize(t: lang.Double, `type`: Type, jsonSerializationContext: JsonSerializationContext): JsonElement = + if (t == t.longValue()) new JsonPrimitive(t.longValue()) else new JsonPrimitive(t) + }).create + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml new file mode 100644 index 000000000..9caf38ff6 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml @@ -0,0 +1,88 @@ + + + + + 4.0.0 + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + + dss-linkis-node-execution + + + + org.apache.linkis + linkis-computation-client + ${linkis.version} + + + + org.apache.linkis + linkis-protocol + ${linkis.version} + + + + org.apache.linkis + linkis-storage-script-dev-client + ${linkis.version} + + + + org.apache.linkis + linkis-label-common + ${linkis.version} + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + 1.8 + 1.8 + UTF-8 + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + + org.apache.maven.plugins + maven-jar-plugin + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java new file mode 100644 index 000000000..d8b1254ed --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.conf; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.Configuration; + +import java.util.Map; + +public class LinkisJobExecutionConfiguration { + + public static final String LINKIS_TYPE = "linkistype"; + + public static final String APPCONN = "appconn"; + + public static final String DSS_LABELS_KEY = "labels"; + + public static final String PROJECT_PREFIX = "project"; + public static final String JOB_PREFIX = "job"; + + public static final String FLOW_VARIABLE_PREFIX = "flow.variable."; + + public static final String NODE_CONF_PREFIX = "node.conf."; + + public static final String CONF_STARTUP = "startup"; + public static final String CONF_RUNTIME = "runtime"; + public static final String CONF_SPECIAL = "special"; + + public static final String RESOURCES_NAME = ".resources"; + + public final static String LINKIS_VERSION_KEY = "linkis.version"; + + public static final String LINKIS_SUBMIT_USER = "wds.dss.workflow.submit.user"; + + public static final String LINKIS_CONTROL_EMPTY_NODE = "linkis.control.empty"; + + public static final String FLOW_CONTEXTID = "wds.linkis.flow.contextID"; + + public final static CommonVars JOB_DEFAULT_TYPE = CommonVars.apply("wds.linkis.flow.default.job.type","linkis"); + + public final static CommonVars LINKIS_DEFAULT_TYPE = CommonVars.apply("wds.linkis.flow.default.linkis.type","spark.sql"); + + public final static CommonVars LINKIS_CONNECTION_TIMEOUT = CommonVars.apply("wds.linkis.flow.connection.timeout",300000); + + public final static CommonVars LINKIS_JOB_REQUEST_STATUS_TIME = CommonVars.apply("wds.linkis.job.status.timeout",10000); + + public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.flow.adminuser","ws"); + + + public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.flow.author.user.token","WS-AUTH"); + + public final static CommonVars LINKIS_JOB_CREATOR = CommonVars.apply("wds.linkis.flow.job.creator","nodeexecution"); + + public final static CommonVars LINKIS_JOB_CREATOR_1_X = CommonVars.apply("wds.linkis.flow.job.creator.v1","Schedulis"); + + public final static CommonVars LINKIS_URL = CommonVars.apply("wds.linkis.gateway.url.v0", Configuration.getGateWayURL()); + + public final static CommonVars LINKIS_URL_1_X = CommonVars.apply("wds.linkis.gateway.url.v1", Configuration.getGateWayURL()); + + public final static CommonVars LINKIS_API_VERSION = CommonVars.apply("wds.linkis.server.version","v1"); + + public final static CommonVars RESULT_PRINT_SIZE = CommonVars.apply("wds.linkis.result.print.size",10); + + + public final static CommonVars LOG_SIZE = CommonVars.apply("wds.linkis.log.size",-1); + public final static CommonVars LOG_ARRAY_LEN = CommonVars.apply("wds.linkis.log.array.len",4); + + public final static CommonVars REQUEST_MAX_RETRY_TIME = CommonVars.apply("wds.linkis.log.retry.time",10); + + public final static CommonVars MAX_HTTP_CONNECTION_COUNT = CommonVars.apply("wds.linkis.job.max.http.connection.count",100); + + //兼容老版本 + public static final CommonVars LINKIS_DEFAULT_VERSION = CommonVars.apply("wds.dss.workflow.execution.linkis.version", "1.0.0"); + + public static final CommonVars LINKIS_DISCOVERY_ENABLE = CommonVars.apply("wds.dss.workflow.execution.linkis.discovery.enable", false); + public static final CommonVars EMBEDDED_FLOW_ID = CommonVars.apply("wds.dss.workflow.execution.subflow.flag","embeddedFlowId"); + public static boolean isLinkis1_X(Map props) { + return props.getOrDefault(LinkisJobExecutionConfiguration.LINKIS_VERSION_KEY,"") + .startsWith("1."); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/BMLResource.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/BMLResource.java new file mode 100644 index 000000000..f9b6fbe75 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/BMLResource.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.entity; + + +public class BMLResource { + + private String resourceId; + private String version; + private String fileName; + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/ContextInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/ContextInfo.java new file mode 100644 index 000000000..5bef46159 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/ContextInfo.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.entity; + + +public class ContextInfo { + + + + private Object value; + + private int readNum; + + + public ContextInfo(Object value, int readNum) { + this.value = value; + this.readNum = readNum; + } + + public Object getValue() { + return value; + } + + public void setValue(Object value) { + this.value = value; + } + + public int getReadNum() { + return readNum; + } + + public void setReadNum(int readNum) { + this.readNum = readNum; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/WorkspaceInfoGetAction.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/WorkspaceInfoGetAction.java new file mode 100644 index 000000000..d3aaee3ea --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/WorkspaceInfoGetAction.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.entity; + +import org.apache.linkis.httpclient.request.GetAction; +import org.apache.linkis.httpclient.request.UserAction; + + +public class WorkspaceInfoGetAction extends GetAction implements UserAction { + + private String url; + private String user; + + @Override + public String getURL() { + return url; + } + + public void setURL(String url) { + this.url = url; + } + + @Override + public void setUser(String user) { + this.user = user; + } + + @Override + public String getUser() { + return user; + } + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionErrorException.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionErrorException.java new file mode 100644 index 000000000..9080bf6f9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionErrorException.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.exception; + + +import org.apache.linkis.common.exception.ErrorException; + + +public class LinkisJobExecutionErrorException extends ErrorException { + public LinkisJobExecutionErrorException(int errCode, String desc) { + super(errCode, desc); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionWarnException.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionWarnException.java new file mode 100644 index 000000000..0b92b43ce --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionWarnException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.exception; + +import org.apache.linkis.common.exception.WarnException; + + +public class LinkisJobExecutionWarnException extends WarnException { + + public LinkisJobExecutionWarnException(int errCode, String desc) { + super(errCode, desc); + } + + public LinkisJobExecutionWarnException(int errCode, String desc, Throwable cause) { + super(errCode, desc); + initCause(cause); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/LinkisNodeExecution.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/LinkisNodeExecution.java new file mode 100644 index 000000000..4c5b4b771 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/LinkisNodeExecution.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.execution; + +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import java.io.Closeable; + + +public interface LinkisNodeExecution extends Closeable { + + void runJob(Job job) throws Exception; + + String getState(Job job); + + String getLog(Job job); + + void waitForComplete(Job job) throws Exception; + + void cancel(Job job) throws Exception; + + double getProgress(Job job); + + Boolean isCompleted(Job job); + + int getResultSize(Job job); + + String getResult(Job job, int index, int maxSize); + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java new file mode 100644 index 000000000..9615f5c11 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java @@ -0,0 +1,352 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.execution.impl; + +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; +import com.webank.wedatasphere.dss.linkis.node.execution.execution.LinkisNodeExecution; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.listener.LinkisExecutionListener; +import com.webank.wedatasphere.dss.linkis.node.execution.parser.CodeParser; +import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobParamsParser; +import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobParser; +import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobRuntimeParamsParser; +import com.webank.wedatasphere.dss.linkis.node.execution.service.LinkisURLService; +import com.webank.wedatasphere.dss.linkis.node.execution.service.impl.BuildJobActionImpl; +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisUjesClientUtils; +import org.apache.linkis.common.exception.LinkisException; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.request.JobExecuteAction; +import org.apache.linkis.ujes.client.request.JobSubmitAction; +import org.apache.linkis.ujes.client.request.OpenLogAction; +import org.apache.linkis.ujes.client.request.ResultSetAction; +import org.apache.linkis.ujes.client.response.JobInfoResult; +import org.apache.linkis.ujes.client.response.JobLogResult; +import org.apache.linkis.ujes.client.response.OpenLogResult; +import org.apache.commons.lang.StringUtils; +import scala.tools.nsc.settings.Final; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; + + +public class LinkisNodeExecutionImpl implements LinkisNodeExecution , LinkisExecutionListener { + + private static LinkisNodeExecution linkisExecution = new LinkisNodeExecutionImpl(); + + private LinkisNodeExecutionImpl() { + registerJobParser(new CodeParser()); + registerJobParser(new JobRuntimeParamsParser()); + registerJobParser(new JobParamsParser()); + } + + public static LinkisNodeExecution getLinkisNodeExecution() { + return linkisExecution; + } + + private final Map clientMap = new HashMap<>(); + + private ArrayList jobParsers = new ArrayList<>(); + + protected UJESClient getClient(Job job) { + Map props =job.getJobProps(); + String linkisUrl = LinkisURLService.Factory.getLinkisURLService().getLinkisURL(job); + if(clientMap.containsKey(linkisUrl)) { + return clientMap.get(linkisUrl); + } + synchronized (clientMap) { + if(!clientMap.containsKey(linkisUrl)) { + UJESClient client = LinkisUjesClientUtils.getUJESClient( + linkisUrl, + LinkisJobExecutionConfiguration.LINKIS_ADMIN_USER.getValue(props), + LinkisJobExecutionConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue(props), + props); + clientMap.put(linkisUrl, client); + job.getLogObj().info("Create a new Linkis client by " + linkisUrl); + return client; + } + } + return clientMap.get(linkisUrl); + } + + public void registerJobParser(JobParser jobParser) { + this.jobParsers.add(jobParser); + } + + @Override + public void runJob(Job job) throws Exception { + // job parser + for(JobParser parser : jobParsers){ + parser.parseJob(job); + } + + Map props =job.getJobProps(); + if(LinkisJobExecutionConfiguration.isLinkis1_X(props)) { + JobSubmitAction submitAction = BuildJobActionImpl.getbuildJobAction().getSubmitAction(job); + job.setJobExecuteResult(getClient(job).submit(submitAction)); + }else{ + //兼容0.X版本的任务提交方式 + JobExecuteAction jobAction = BuildJobActionImpl.getbuildJobAction().getJobAction(job); + job.setJobExecuteResult(getClient(job).execute(jobAction)); + } + + job.getLogObj().info("<---------------Start to execute job--------------->"); + job.getLogObj().info("Task id is:"+ job.getJobExecuteResult().getTaskID()); + job.getLogObj().info("Exec id is:"+ job.getJobExecuteResult().getExecID()); + } + + @Override + public String getState(Job job) { + return getClient(job).getJobInfo(job.getJobExecuteResult()).getJobStatus(); + } + + @Override + public String getLog(Job job) { + + JobInfoResult jobInfo =getClient(job).getJobInfo(job.getJobExecuteResult()); + + List logArray = null; + //只有job执行失败了,才打印全部日志,否则只打印linkis运行时缓存日志 + if(jobInfo.isCompleted() && jobInfo.isFailed()){ + try { + logArray = Arrays.asList(queryPersistedLogAll(job).getLog()); + } catch (LinkisJobExecutionErrorException e) { + job.getLogObj().error("Get full log failed:"+e.getMessage()); + } + }else if(jobInfo.isRunning()){ + JobLogResult jobLogResult = getClient(job) + .log(job.getJobExecuteResult(), + job.getLogFromLine(), + LinkisJobExecutionConfiguration.LOG_SIZE.getValue()); + + job.setLogFromLine(jobLogResult.fromLine()); + logArray = jobLogResult.getLog(); + }else { + job.getLogObj().info("Job run is completed and the cache log can not be printed "); + } + + if (logArray != null && logArray.size() + >= LinkisJobExecutionConfiguration.LOG_ARRAY_LEN.getValue() + && StringUtils.isNotEmpty(logArray.get(3))) { + return logArray.get(3); + } + return null; + } + + + + + + public OpenLogResult queryPersistedLogAll(Job job) throws LinkisJobExecutionErrorException { + String taskID = job.getJobExecuteResult().getTaskID(); + String user = job.getUser(); + JobInfoResult jobInfo = getClient(job).getJobInfo(job.getJobExecuteResult()); + String logPath = jobInfo.getRequestPersistTask().getLogPath(); + int retryCnt = 0; + final int MAX_RETRY_TIMES = LinkisJobExecutionConfiguration.REQUEST_MAX_RETRY_TIME.getValue(); + OpenLogResult openLogResult = null; + + int backCnt = 0; + final int MAX_BACK_TIMES = 3; + + while (retryCnt++ < MAX_RETRY_TIMES) { + try { + openLogResult = getClient(job).openLog(OpenLogAction.newBuilder().setLogPath(logPath).setProxyUser(user).build()); + //job.getLogObj().info("persisted-log-result:" + LinkisJobExecutionUtils.gson.toJson(openLogResult)); + if (openLogResult == null || + 0 != openLogResult.getStatus() || + StringUtils.isBlank(openLogResult.getLog()[LinkisJobExecutionUtils.IDX_FOR_LOG_TYPE_ALL])) { + String reason; + if (openLogResult == null) { + reason = "OpenLogResult is null"; + } else if (0 != openLogResult.getStatus()) { + reason = "server returns non-zero status-code"; + } else { + reason = "server returns empty log"; + } + String msg = MessageFormat.format("Get log from openLog failed. retry time : {0}/{1}. taskID={2}. Reason: {3}", retryCnt, MAX_RETRY_TIMES, taskID, reason); + job.getLogObj().warn(msg); + if (retryCnt >= MAX_RETRY_TIMES) { + if (backCnt >= MAX_BACK_TIMES) { + msg = MessageFormat.format("Get log from openLog failed. Retry exhausted. taskID={0}, Reason: {1}", job.getJobExecuteResult().getTaskID(), reason); + throw new LinkisJobExecutionErrorException(100079,msg); + } else { + backCnt++; + retryCnt = 0; + Utils.sleepQuietly(LinkisJobExecutionConfiguration.LINKIS_JOB_REQUEST_STATUS_TIME.getValue(job.getJobProps()));;//wait 10s and try again + } + } + } else { + break; + } + } catch (Exception e) { + String msg = MessageFormat.format("Get log from openLog failed. retry time : {0}/{1}", retryCnt, MAX_RETRY_TIMES); + if (e instanceof LinkisException) { + msg += " " + e.toString(); + } + job.getLogObj().warn(msg, e); + if (retryCnt >= MAX_RETRY_TIMES) { + if (backCnt >= MAX_BACK_TIMES) { + throw new LinkisJobExecutionErrorException(100080, "Get log from openLog failed. Retry exhausted. taskID=" + taskID); + } else { + backCnt++; + retryCnt = 0; + Utils.sleepQuietly(LinkisJobExecutionConfiguration.LINKIS_JOB_REQUEST_STATUS_TIME.getValue(job.getJobProps()));//wait 10s and try again + } + } + } + Utils.sleepQuietly(LinkisJobExecutionConfiguration.LINKIS_JOB_REQUEST_STATUS_TIME.getValue(job.getJobProps()));; + } + return openLogResult; + } + + + + + @Override + public void waitForComplete(Job job) throws Exception { + JobInfoResult jobInfo = getClient(job).getJobInfo(job.getJobExecuteResult()); + int count = 0; + while (!jobInfo.isCompleted()) { + double progress = -1; + try{ + progress = this.getProgress(job); + }catch(Exception e){ + //ignore + } + if (progress >= 0){ + job.getLogObj().info("Update Progress info: " + progress); + } + JobInfoResult oldJobInfo = jobInfo; + try{ + jobInfo = getClient(job).getJobInfo(job.getJobExecuteResult()); + }catch(Throwable e){ + jobInfo = oldJobInfo; + count += 1; + job.getLogObj().error("不能获取到正确的状态,计数 count = " + count); + //两分钟内获取不到,就认为不行,因为这个时候我应该是重启完成了 + if (count == 40) { + job.getLogObj().info("超过40次不能获取状态,应该是linkis不能获取到正常信息,判断任务失败"); + throw new LinkisJobExecutionErrorException(90101, "Failed to execute Job: " + e.getMessage()); + } + } + Utils.sleepQuietly(LinkisJobExecutionConfiguration.LINKIS_JOB_REQUEST_STATUS_TIME.getValue(job.getJobProps())); + printJobLog(job); + } + if (!jobInfo.isSucceed()) { + Utils.sleepQuietly(LinkisJobExecutionConfiguration.LINKIS_JOB_REQUEST_STATUS_TIME.getValue(job.getJobProps())); + printJobLog(job); + throw new LinkisJobExecutionErrorException(90101, "Failed to execute Job: " + jobInfo.getMessage()); + } + } + + private void printJobLog(Job job){ + String log = null; + try{ + log = this.getLog(job); + }catch(Exception e){ + job.getLogObj().warn("failed to get log info", e); + } + if (StringUtils.isEmpty(log)){ + return; + } + Arrays.stream(log.split("\n")).forEach(l -> { + if (l != null) { + if(l.contains("ERROR")) { + job.getLogObj().error(l); + }else{ + job.getLogObj().info(l); + } + } + }); + + } + + @Override + public void cancel(Job job) throws Exception { + try { + getClient(job).kill(job.getJobExecuteResult()); + } catch (Exception e) { + job.getLogObj().error("linkis execute kill operation failed,reason:" + e.getMessage() + ""); + } + } + + @Override + public double getProgress(Job job) { + return getClient(job).progress(job.getJobExecuteResult()).getProgress(); + } + + @Override + public Boolean isCompleted(Job job) { + return getClient(job).getJobInfo(job.getJobExecuteResult()).isCompleted(); + } + + @Override + public int getResultSize(Job job) { + JobInfoResult jobInfo = getClient(job).getJobInfo(job.getJobExecuteResult()); + + job.getLogObj().info("JobInfo result location is "+jobInfo.getRequestPersistTask().getResultLocation()); + + job.getLogObj().info("JobInfo user location is "+jobInfo.getRequestPersistTask().getUmUser()); + //解决azkaban 执行报 user is need问题 + jobInfo.getRequestPersistTask().setUmUser(job.getUser()); + if (jobInfo.isSucceed()) { + String[] resultSetList = jobInfo.getResultSetList(getClient(job)); + if (resultSetList != null && resultSetList.length > 0) { + return resultSetList.length; + } + } + return 0; + } + + @Override + public String getResult(Job job, int index, int maxSize) { + String resultContent = null; + JobInfoResult jobInfo = getClient(job).getJobInfo(job.getJobExecuteResult()); + String[] resultSetList = jobInfo.getResultSetList(getClient(job)); + if (resultSetList != null && resultSetList.length > 0) { + Object fileContent = getClient(job).resultSet(ResultSetAction.builder() + .setPath(resultSetList[index]) + .setUser(job.getJobExecuteResult().getUser()) + .setPageSize(maxSize).build()).getFileContent(); + if (fileContent instanceof ArrayList) { + ArrayList> resultSetRow = (ArrayList>) fileContent; + resultContent = StringUtils.join(resultSetRow, "\n"); + } else { + resultContent = fileContent.toString(); + } + } + return resultContent; + } + + @Override + public void onStatusChanged(String fromState, String toState, Job job) { + } + + @Override + public void close() { + clientMap.values().forEach(IOUtils::closeQuietly); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppConnLinkisJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppConnLinkisJob.java new file mode 100644 index 000000000..5577b9cd1 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppConnLinkisJob.java @@ -0,0 +1,192 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; +import org.apache.linkis.ujes.client.response.JobExecuteResult; + +import java.util.Map; + + +public abstract class AbstractAppConnLinkisJob extends AppConnLinkisJob { + + private Map jobProps; + private Map source; + + private JobTypeEnum jobType; + + private Map variables; + + private Map configuration; + + private String code; + + private String engineType; + + private String runType; + + + + private Map params; + + private Map runtimeParams; + + private JobExecuteResult jobExecuteResult; + + private LinkisJobExecutionLog logObj; + + private int logFromLine; + + + + + + + @Override + public Map getSource() { + return this.source; + } + + @Override + public JobTypeEnum getJobType() { + return this.jobType; + } + + + @Override + public Map getVariables() { + return this.variables; + } + + @Override + public Map getConfiguration() { + return this.configuration; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public void setCode(String code) { + this.code = code; + } + + @Override + public String getEngineType() { + return this.engineType; + } + + @Override + public String getRunType() { + return this.runType; + } + + + + @Override + public Map getParams() { + return this.params; + } + + @Override + public Map getRuntimeParams() { + return this.runtimeParams; + } + + @Override + public JobExecuteResult getJobExecuteResult() { + return this.jobExecuteResult; + } + + @Override + public void setJobExecuteResult(JobExecuteResult jobExecuteResult) { + this.jobExecuteResult = jobExecuteResult; + } + + @Override + public Map getJobProps() { + return this.jobProps; + } + + @Override + public void setJobProps(Map jobProps) { + this.jobProps = jobProps; + } + + @Override + public LinkisJobExecutionLog getLogObj() { + return this.logObj; + } + + @Override + public int getLogFromLine() { + return this.logFromLine; + } + + @Override + public void setLogFromLine(int index) { + this.logFromLine = index; + } + + + @Override + public void setSource(Map source) { + this.source = source; + } + + @Override + public void setJobType(JobTypeEnum jobType) { + this.jobType = jobType; + } + + @Override + public void setVariables(Map variables) { + this.variables = variables; + } + + @Override + public void setConfiguration(Map configuration) { + this.configuration = configuration; + } + + @Override + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + @Override + public void setRunType(String runType) { + this.runType = runType; + } + + @Override + public void setParams(Map params) { + this.params = params; + } + + @Override + public void setRuntimeParams(Map runtimeParams) { + this.runtimeParams = runtimeParams; + } + + @Override + public void setLogObj(LinkisJobExecutionLog logObj) { + this.logObj = logObj; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java new file mode 100644 index 000000000..a5448eef4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java @@ -0,0 +1,236 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; +import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; +import org.apache.linkis.ujes.client.response.JobExecuteResult; + +import java.util.List; +import java.util.Map; + + +public abstract class AbstractCommonLinkisJob extends CommonLinkisJob { + + private Map jobProps; + + + + private Map source; + + private JobTypeEnum jobType; + + private Map variables; + + private Map configuration; + + private String code; + + private String engineType; + + private String runType; + + + + private Map params; + + private Map runtimeParams; + + private JobExecuteResult jobExecuteResult; + + private LinkisJobExecutionLog logObj; + + private int logFromLine; + + + private List jobResourceList; + + private List projectResourceList; + + private Map> fLowNameAndResources; + + + + + @Override + public Map getSource() { + return this.source; + } + + @Override + public JobTypeEnum getJobType() { + return this.jobType; + } + + + @Override + public Map getVariables() { + return this.variables; + } + + @Override + public Map getConfiguration() { + return this.configuration; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public void setCode(String code) { + this.code = code; + } + + @Override + public String getEngineType() { + return this.engineType; + } + + @Override + public String getRunType() { + return this.runType; + } + + + + @Override + public Map getParams() { + return this.params; + } + + @Override + public Map getRuntimeParams() { + return this.runtimeParams; + } + + @Override + public JobExecuteResult getJobExecuteResult() { + return this.jobExecuteResult; + } + + @Override + public void setJobExecuteResult(JobExecuteResult jobExecuteResult) { + this.jobExecuteResult = jobExecuteResult; + } + + @Override + public Map getJobProps() { + return this.jobProps; + } + + @Override + public void setJobProps(Map jobProps) { + this.jobProps = jobProps; + } + + @Override + public LinkisJobExecutionLog getLogObj() { + return this.logObj; + } + + @Override + public int getLogFromLine() { + return this.logFromLine; + } + + @Override + public void setLogFromLine(int index) { + this.logFromLine = index; + } + + + @Override + public void setSource(Map source) { + this.source = source; + } + + @Override + public void setJobType(JobTypeEnum jobType) { + this.jobType = jobType; + } + + @Override + public void setVariables(Map variables) { + this.variables = variables; + } + + @Override + public void setConfiguration(Map configuration) { + this.configuration = configuration; + } + + @Override + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + @Override + public void setRunType(String runType) { + this.runType = runType; + } + + @Override + public void setParams(Map params) { + this.params = params; + } + + @Override + public void setRuntimeParams(Map runtimeParams) { + this.runtimeParams = runtimeParams; + } + + @Override + public void setLogObj(LinkisJobExecutionLog logObj) { + this.logObj = logObj; + } + + + @Override + public List getJobResourceList() { + return this.jobResourceList; + } + + + + @Override + public List getProjectResourceList() { + return this.projectResourceList; + } + + @Override + public Map> getFlowNameAndResources() { + return this.fLowNameAndResources; + } + + @Override + public void setJobResourceList(List jobResourceList) { + this.jobResourceList = jobResourceList; + } + + @Override + public void setProjectResourceList(List projectResourceList) { + this.projectResourceList = projectResourceList; + } + + @Override + public void setFlowNameAndResources(Map> fLowNameAndResources) { + this.fLowNameAndResources = fLowNameAndResources; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AppConnLinkisJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AppConnLinkisJob.java new file mode 100644 index 000000000..48e18a546 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AppConnLinkisJob.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + + +public abstract class AppConnLinkisJob extends LinkisJob { +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Builder.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Builder.java new file mode 100644 index 000000000..bc36a4a7b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Builder.java @@ -0,0 +1,148 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.entity.WorkspaceInfoGetAction; +import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; +import com.webank.wedatasphere.dss.linkis.node.execution.service.LinkisURLService; +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisUjesClientUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.utils.JsonUtils; +import org.apache.linkis.httpclient.dws.DWSHttpClient; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.response.impl.DefaultHttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + + +public abstract class Builder { + + protected Logger logger = LoggerFactory.getLogger(getClass()); + + protected abstract String getJobType(); + + protected abstract LinkisJob creatLinkisJob(boolean isLinkisType); + + protected abstract void fillJobInfo(Job job); + + protected abstract void fillLinkisJobInfo(LinkisJob linkisJob); + + protected abstract void fillCommonLinkisJobInfo(CommonLinkisJob linkisAppConnJob); + + + public Job build() throws Exception { + + LinkisJob job = null; + String jobType = getJobType(); + String[] jobTypeSplit = jobType.split("\\."); + if (jobTypeSplit.length < 2) { + throw new LinkisJobExecutionErrorException(90100, "This is not Linkis job type, this jobtype is " + jobType); + } + String engineType = jobTypeSplit[1]; + //delete linkis.engineType + String runType = StringUtils.substringAfterLast(jobType, jobTypeSplit[0] + "." + jobTypeSplit[1] + "."); + + if (LinkisJobExecutionConfiguration.LINKIS_CONTROL_EMPTY_NODE.equalsIgnoreCase(jobType)) { + job = new AbstractCommonLinkisJob() { + @Override + public String getSubmitUser() { + return null; + } + + @Override + public String getUser() { + return null; + } + + @Override + public String getJobName() { + return null; + } + }; + + job.setJobType(JobTypeEnum.EmptyJob); + return job; + } + if (LinkisJobExecutionUtils.isCommonAppConnJob(engineType)) { + job = creatLinkisJob(false); + job.setJobType(JobTypeEnum.CommonJob); + } else { + job = creatLinkisJob(true); + job.setJobType(JobTypeEnum.CommonJob); + fillCommonLinkisJobInfo((CommonLinkisJob) job); + } + + job.setEngineType(engineType); + job.setRunType(runType); + fillJobInfo(job); + fillLinkisJobInfo(job); + + if (job.getRuntimeParams() == null) { + job.setRuntimeParams(new HashMap<>()); + } + String contextId = getContextID(job); + if (StringUtils.isBlank(contextId)) { + throw new LinkisJobExecutionErrorException(90100, "contextID is not exists."); + } + contextId = contextId.replace("/", "\\"); + Map contextMap = JsonUtils.jackson().readValue(contextId, Map.class); + logger.info("the contextMap is:{}", contextMap); + String workspaceName = (String) JsonUtils.jackson().readValue((String) contextMap.get("value"), Map.class).get("workspace"); + logger.info("try to get workspace str by workspaceName {}.", workspaceName); + String workspace = getWorkspaceStr(job, workspaceName); + logger.info("Got workspace str {}.", workspace); + job.getRuntimeParams().put("contextID", contextId); + job.getRuntimeParams().put("workspace", workspace); + return job; + } + + protected abstract String getContextID(Job job); + + private String getWorkspaceStr(Job job, String workspaceName) throws Exception { + String user = job.getUser(); + String linkisUrl = LinkisURLService.Factory.getLinkisURLService().getDefaultLinkisURL(job); + String token = LinkisJobExecutionConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue(job.getJobProps()); + DWSHttpClient client = null; + DWSClientConfig clientConfig = LinkisUjesClientUtils.getClientConfig1_X(linkisUrl, user, token, job.getJobProps()); + WorkspaceInfoGetAction workspaceInfoGetAction = new WorkspaceInfoGetAction(); + workspaceInfoGetAction.setURL("/api/rest_j/v1/dss/framework/workspace/getWorkSpaceStr"); + workspaceInfoGetAction.setParameter("workspaceName", workspaceName); + workspaceInfoGetAction.setUser(user); + workspaceInfoGetAction.addHeader("Referer", ""); + try { + client = new DWSHttpClient(clientConfig, "Workspace-Fetch-Client-"); + DefaultHttpResult result = (DefaultHttpResult) client.execute(workspaceInfoGetAction); + if (result.getStatusCode() == 200 || result.getStatusCode() == 0) { + Map responseBody = JsonUtils.jackson().readValue(result.getResponseBody(), Map.class); + Map data = (Map) responseBody.get("data"); + return (String) data.get("workspaceStr"); + } else { + throw new LinkisJobExecutionErrorException(50063, "Failed to get workspace str, responseBody is: " + + result.getResponseBody()); + } + } finally { + IOUtils.closeQuietly(client); + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/CommonLinkisJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/CommonLinkisJob.java new file mode 100644 index 000000000..cac95a127 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/CommonLinkisJob.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + +public abstract class CommonLinkisJob extends LinkisJob { + + public abstract List getJobResourceList(); + + public abstract void setJobResourceList(List jobResourceList); + + public abstract List getProjectResourceList(); + + public abstract void setProjectResourceList(List projectResourceList); + + public abstract Map> getFlowNameAndResources(); + + public abstract void setFlowNameAndResources(Map> fLowNameAndResources); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java new file mode 100644 index 000000000..55b2ce9d0 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; +import org.apache.linkis.ujes.client.response.JobExecuteResult; + +import java.util.Map; + +public interface Job { + + String getCode(); + + void setCode(String code); + + String getEngineType(); + + void setEngineType(String engineType); + + String getRunType(); + + void setRunType(String runType); + + String getUser(); + + String getJobName(); + + Map getParams(); + + void setParams(Map params); + + Map getRuntimeParams(); + void setRuntimeParams(Map runtimeParams); + + JobExecuteResult getJobExecuteResult(); + + void setJobExecuteResult(JobExecuteResult jobExecuteResult); + + Map getJobProps(); + + void setJobProps(Map jobProps); + + LinkisJobExecutionLog getLogObj(); + void setLogObj(LinkisJobExecutionLog logObj); + + int getLogFromLine(); + + void setLogFromLine(int index); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobTypeEnum.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobTypeEnum.java new file mode 100644 index 000000000..813527db2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobTypeEnum.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + + +public enum JobTypeEnum { + CommonJob, EmptyJob +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/LinkisJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/LinkisJob.java new file mode 100644 index 000000000..ea6b20dcf --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/LinkisJob.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + + +import java.util.Map; + + +public abstract class LinkisJob implements Job{ + + + public abstract Map getSource(); + + public abstract void setSource(Map source); + + public abstract JobTypeEnum getJobType(); + public abstract void setJobType(JobTypeEnum jobType); + + public abstract String getSubmitUser(); + + public abstract Map getVariables(); + public abstract void setVariables(Map variables); + + + public abstract Map getConfiguration(); + public abstract void setConfiguration(Map configuration); + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/RuntimeJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/RuntimeJob.java new file mode 100644 index 000000000..3cf3cb4a2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/RuntimeJob.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + + +public interface RuntimeJob { +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/listener/LinkisExecutionListener.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/listener/LinkisExecutionListener.java new file mode 100644 index 000000000..403bb9a55 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/listener/LinkisExecutionListener.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.listener; + +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; + + +public interface LinkisExecutionListener { + + void onStatusChanged(String fromState, String toState, Job job); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/log/LinkisJobExecutionLog.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/log/LinkisJobExecutionLog.java new file mode 100644 index 000000000..5aad912a5 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/log/LinkisJobExecutionLog.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.log; + + +public abstract class LinkisJobExecutionLog { + + public void info(Object message){ + info(message, null); + } + public abstract void info(Object message, Throwable t); + + public void warn(Object message){ + warn(message, null); + } + public abstract void warn(Object message, Throwable t); + + public void error(Object message){ + error(message, null); + } + public abstract void error(Object message, Throwable t); + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/CodeParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/CodeParser.java new file mode 100644 index 000000000..cecc1cf17 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/CodeParser.java @@ -0,0 +1,249 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.parser; + +import com.google.gson.reflect.TypeToken; +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; +import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; +import com.webank.wedatasphere.dss.linkis.node.execution.job.CommonLinkisJob; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.service.LinkisURLService; +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; +import org.apache.linkis.filesystem.WorkspaceClientFactory; +import org.apache.linkis.filesystem.request.WorkspaceClient; +import org.apache.linkis.filesystem.response.ScriptFromBMLResponse; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; + + +public class CodeParser implements JobParser { + + private static final Pattern pb = Pattern.compile("((project)|(flow)|(node))://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE); + private WorkspaceClient client1X = null; + private WorkspaceClient client0X = null; + private final Object clientLocker = new Object(); + @Override + public void parseJob(Job job) throws Exception{ + if (! ( job instanceof CommonLinkisJob) ) { + return ; + } + CommonLinkisJob linkisAppConnJob = (CommonLinkisJob) job; + Map script = LinkisJobExecutionUtils.gson.fromJson(linkisAppConnJob.getCode(), new TypeToken>() {}.getType()); + List jobResourceList = linkisAppConnJob.getJobResourceList(); + BMLResource scriptResource = null; + if (script == null) { + throw new LinkisJobExecutionErrorException(90102,"Script is empty"); + } + String fileName = (String) script.get("script"); + for(BMLResource bmlResource : jobResourceList){ + if(bmlResource.getFileName().equals(fileName)){ + scriptResource = bmlResource; + break; + } + } + if(null == scriptResource) { + throw new LinkisJobExecutionErrorException(90102,"Failed to get script resource"); + } + + Map executionParams = getExecutionParams(scriptResource, linkisAppConnJob); + if (executionParams.get("executionCode") != null) { + String executionCode = (String) executionParams.get("executionCode"); + linkisAppConnJob.getLogObj().info("************************************SUBMIT CODE************************************"); + linkisAppConnJob.getLogObj().info(executionCode); + linkisAppConnJob.getLogObj().info("************************************SUBMIT CODE************************************"); + //parsedCode + linkisAppConnJob.setCode(executionCode); + } + if (executionParams.get("params") != null && executionParams.get("params") instanceof Map) { + if (linkisAppConnJob.getParams() != null) { + linkisAppConnJob.getParams().putAll( (Map)executionParams.get("params")); + } + } + } + + private Map getExecutionParams(BMLResource bmlResource, CommonLinkisJob linkisAppConnJob) { + Map map = new HashMap<>(); + ScriptFromBMLResponse response = getOrCreateWorkSpaceClient(linkisAppConnJob).requestOpenScriptFromBML(bmlResource.getResourceId(), bmlResource.getVersion(), bmlResource.getFileName()); + linkisAppConnJob.getLogObj().info("Get execution code from workspace client,bml resource id "+bmlResource.getResourceId()+", version is "+bmlResource.getVersion()); + map.put("executionCode", response.scriptContent()); + map.put("params", response.metadata()); + return map; + } + + private WorkspaceClient getOrCreateWorkSpaceClient(CommonLinkisJob linkisAppConnJob) { + Map props = linkisAppConnJob.getJobProps(); + if(LinkisJobExecutionConfiguration.isLinkis1_X(props)) { + if (null == client1X) { + synchronized (clientLocker) { + if (null == client1X) { + this.client1X = WorkspaceClientFactory.getClient(linkisAppConnJob.getSubmitUser(), LinkisJobExecutionConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue(linkisAppConnJob.getJobProps()), + LinkisURLService.Factory.getLinkisURLService().getLinkisURL(linkisAppConnJob)); + } + } + } + linkisAppConnJob.getLogObj().info("Use workspace client1X:"+LinkisURLService.Factory.getLinkisURLService().getLinkisURL(linkisAppConnJob)); + return client1X; + }else{ + if (null == client0X) { + synchronized (clientLocker) { + if (null == client0X) { + this.client0X = WorkspaceClientFactory.getClient(linkisAppConnJob.getSubmitUser(), LinkisJobExecutionConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue(linkisAppConnJob.getJobProps()), + LinkisURLService.Factory.getLinkisURLService().getLinkisURL(linkisAppConnJob)); + } + } + } + linkisAppConnJob.getLogObj().info("Use workspace client0X:"+LinkisURLService.Factory.getLinkisURLService().getLinkisURL(linkisAppConnJob)); + return client0X; + } + } + + private ArrayList getResourceNames(String code){ + ArrayList bmlResourceNames = new ArrayList(); + Matcher mb = pb.matcher(code); + while (mb.find()) { + bmlResourceNames.add(mb.group().trim()); + } + return bmlResourceNames; + } + + + /** + * 1.Find the project file used in the script + * 2.Find the node file used in the script + * 3.Recursively find the flow file used in the script + * 4.Replace file name with prefixed name + * @param resourceNames + * @param linkisAppConnJob + * @return + */ + private ArrayList getResourcesByNames(ArrayList resourceNames, CommonLinkisJob linkisAppConnJob) { + + ArrayList bmlResourceArrayList = new ArrayList<>(); + + String jobName = linkisAppConnJob.getJobName(); + String flowName = linkisAppConnJob.getSource().get("flowName"); + String projectName = linkisAppConnJob.getSource().get("projectName"); + + + List projectResourceList = linkisAppConnJob.getProjectResourceList(); + + + List jobResourceList = linkisAppConnJob.getJobResourceList(); + for (String resourceName : resourceNames) { + String[] resourceNameSplit = resourceName.split("://"); + String prefix = resourceNameSplit[0].toLowerCase(); + String fileName = resourceNameSplit[1]; + BMLResource resource = null; + String afterFileName = fileName; + switch (prefix) { + case "project": + resource = findResource(projectResourceList, fileName); + afterFileName = LinkisJobExecutionConfiguration.PROJECT_PREFIX + "_" + projectName + "_" + fileName; + break; + case "flow": + resource = findFlowResource(linkisAppConnJob, fileName, flowName); + break; + case "node": + resource = findResource(jobResourceList, fileName); + afterFileName = LinkisJobExecutionConfiguration.JOB_PREFIX + "_" + jobName + "_" + fileName; + break; + default: + } + if (null == resource) { + linkisAppConnJob.getLogObj().error("Failed to find the " + prefix + " resource file of " + fileName); + throw new RuntimeException("Failed to find the " + prefix + " resource file of " + fileName); + } + if (!afterFileName.equals(fileName)) { + resource.setFileName(afterFileName); + } + bmlResourceArrayList.add(resource); + } + return bmlResourceArrayList; + } + + + /** + * Recursively find the flow file used in the script + * Recursive exit condition is top-level flow + * + */ + private BMLResource findFlowResource(CommonLinkisJob linkisAppConnJob, String fileName, String flowName) { + + String fullFlowName = ""; + Map> fLowNameAndResources = linkisAppConnJob.getFlowNameAndResources(); + if (fLowNameAndResources == null){ + return null; + } + Optional>> first = fLowNameAndResources.entrySet().stream().filter(fLowNameAndResource -> fLowNameAndResource.getKey().endsWith(flowName + LinkisJobExecutionConfiguration.RESOURCES_NAME)).findFirst(); + + if(first.isPresent()){ + fullFlowName = first.get().getKey(); + BMLResource resource = findResource(first.get().getValue(), fileName); + if (resource != null) { + resource.setFileName(flowName + "_" + fileName); + return resource; + } + } + + String firstFlow = "flow." + flowName + LinkisJobExecutionConfiguration.RESOURCES_NAME; + if (firstFlow.equals(fullFlowName)) { + return null; + } + //getParentFlowName:flow.flows1.test.resources return:flows1 + String parentFlowName = StringUtils.substringAfterLast(StringUtils.substringBefore(fullFlowName, "." + flowName + + LinkisJobExecutionConfiguration.RESOURCES_NAME), "."); + if (StringUtils.isEmpty(parentFlowName)) { + return null; + } + + return findFlowResource(linkisAppConnJob, fileName, parentFlowName); + } + + + private String replaceCodeResourceNames(String code, ArrayList resourceNameList, ArrayList resourceList){ + if(resourceList.size() != resourceNameList.size()){ + throw new RuntimeException("Failed to parsed resource file"); + } + + String[] names = resourceNameList.toArray(new String[]{}); + + String[] afterNames = new String[resourceList.size()]; + for (int i=0 ; i < afterNames.length ; i++){ + afterNames[i] = resourceList.get(i).getFileName(); + } + return StringUtils.replaceEach(code, names, afterNames); + } + + private BMLResource findResource(List resourceArrayList, String fileName){ + if(resourceArrayList != null && !resourceArrayList.isEmpty()) { + for(BMLResource resource : resourceArrayList){ + if(resource.getFileName().equals(fileName)){ + return resource; + } + } + } + return null; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java new file mode 100644 index 000000000..f5072c78e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.parser; + +import com.webank.wedatasphere.dss.linkis.node.execution.job.AppConnLinkisJob; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; +import org.apache.linkis.protocol.utils.TaskUtils; + +import java.util.Map; + + +public class JobParamsParser implements JobParser { + + @Override + public void parseJob(Job job) { + + if (job instanceof LinkisJob) { + LinkisJob linkisJob = (LinkisJob) job; + linkisJob.getLogObj().info("Start to put variable and configuration"); + //put variable + Map flowVariables = linkisJob.getVariables(); + putParamsMap(job.getParams(), "variable", flowVariables); + // put configuration + Map configuration = linkisJob.getConfiguration(); + putParamsMap(job.getParams(), "configuration", configuration); + + linkisJob.getLogObj().info("Finished to put variable and configuration"); + } + // set the variable to runtimeMap since AppConn may need it. + if(job instanceof AppConnLinkisJob) { + job.getRuntimeParams().put("variables", TaskUtils.getVariableMap(job.getParams())); + } + } + + private void putParamsMap(Map params, String key, Map value) { + if (null == value) { + return; + } + if (params.get(key) != null) { + ((Map) params.get(key)).putAll(value); + } else { + params.put(key, value); + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParser.java new file mode 100644 index 000000000..ef4421e42 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParser.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.parser; + +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; + + +public interface JobParser { + + void parseJob(Job job) throws Exception; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobRuntimeParamsParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobRuntimeParamsParser.java new file mode 100644 index 000000000..8fa5f40bb --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobRuntimeParamsParser.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.parser; + +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; + + +public class JobRuntimeParamsParser implements JobParser{ + + @Override + public void parseJob(Job job) { + if(job instanceof LinkisJob) { + job.getRuntimeParams().put(LinkisJobExecutionConfiguration.LINKIS_SUBMIT_USER,((LinkisJob) job).getSubmitUser()); + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/BuildJobAction.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/BuildJobAction.java new file mode 100644 index 000000000..7d0cd692d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/BuildJobAction.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.service; + + +import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import org.apache.linkis.ujes.client.request.JobExecuteAction; +import org.apache.linkis.ujes.client.request.JobSubmitAction; + + +public interface BuildJobAction { + + JobExecuteAction getJobAction(Job job) throws LinkisJobExecutionErrorException; + + JobSubmitAction getSubmitAction(Job job) throws LinkisJobExecutionErrorException; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/LinkisURLService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/LinkisURLService.java new file mode 100644 index 000000000..aaaa896bc --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/LinkisURLService.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.service; + +import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionWarnException; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.service.impl.LinkisURLServiceImpl; +import org.apache.linkis.common.utils.ClassUtils; +import java.util.Optional; + + +public interface LinkisURLService { + + /** + * Get Linkis gateway url by job. If multi-Linkis cluster is needed, user can override this method to provide different urls by job. + * @param job this nodeJob + * @return a accessiable Linkis gateway url. + */ + String getLinkisURL(Job job); + + /** + * Get default Linkis gateway url by job. + * @param job this nodeJob + * @return a accessiable Linkis gateway url. + */ + String getDefaultLinkisURL(Job job); + + class Factory { + + private Factory() { + } + + private static LinkisURLService linkisURLService; + + public static LinkisURLService getLinkisURLService() { + if(linkisURLService != null) { + return linkisURLService; + } + synchronized (Factory.class) { + if(linkisURLService == null) { + Optional> clazz = ClassUtils.reflections().getSubTypesOf(LinkisURLService.class).stream().filter(c -> !ClassUtils.isInterfaceOrAbstract(c) && + !LinkisURLServiceImpl.class.isAssignableFrom(c)).findFirst(); + if(clazz.isPresent()) { + try { + linkisURLService = clazz.get().newInstance(); + } catch (Exception e) { + throw new LinkisJobExecutionWarnException(24335, "Get LinkisURLService failed!", e); + } + } else { + linkisURLService = new LinkisURLServiceImpl(); + } + } + } + return linkisURLService; + } + + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java new file mode 100644 index 000000000..956d03d0f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java @@ -0,0 +1,207 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.service.impl; + +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; +import com.webank.wedatasphere.dss.linkis.node.execution.service.BuildJobAction; +import org.apache.commons.lang.SerializationUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.manager.label.constant.LabelKeyConstant; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator; +import org.apache.linkis.protocol.utils.TaskUtils; +import org.apache.linkis.ujes.client.request.JobExecuteAction; +import org.apache.linkis.ujes.client.request.JobSubmitAction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +import static com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration.*; + + +public class BuildJobActionImpl implements BuildJobAction { + + private Logger logger = LoggerFactory.getLogger(BuildJobActionImpl.class); + private static BuildJobAction buildJobAction = new BuildJobActionImpl(); + + private BuildJobActionImpl() { + + } + + public static BuildJobAction getbuildJobAction() { + return buildJobAction; + } + + private Map prepareYarnLabel(Job job) { + Map labels = new HashMap<>(); + + Object labelStr = job.getRuntimeParams().getOrDefault(LinkisJobExecutionConfiguration.DSS_LABELS_KEY, null); + + if (StringUtils.isNotBlank((String) labelStr)) { + labels.put(LinkisJobExecutionConfiguration.DSS_LABELS_KEY, labelStr); + logger.info("Set Yarn cluster label is: " + labels.toString()); + } else { + labels.put(LinkisJobExecutionConfiguration.DSS_LABELS_KEY, "DEV-DEV"); + logger.warn("Job Label is null. Then set default value"); + } + return labels; + } + + private String parseExecutionCode(Job job) { + String code = job.getCode(); + logger.info("The parseExecutionCode0X code for the job is {}", code); + if (StringUtils.isEmpty(code) || code.equalsIgnoreCase("null")) { + Gson gson = new Gson(); + code = gson.toJson(job.getParams()); + logger.info("The executable code for the job is {}", code); + } + return code; + } + + + private String parseExecutionCodeFor1X(Job job) { + String code = job.getCode(); + logger.info("The parseExecutionCodeFor1X code for the job is {}", code); + //for appconn node in subflow contains embeddedFlowId + if (StringUtils.isEmpty(code) || code.equalsIgnoreCase("null") || code.contains(EMBEDDED_FLOW_ID.getValue())) { + Gson gson = new Gson(); + code = gson.toJson(job.getParams()); + logger.info("The executable code for the job is {}", code); + } + return code; + } + + + @Override + public JobExecuteAction getJobAction(Job job) throws LinkisJobExecutionErrorException { + + enrichParams(job); + +// Map labels = prepareYarnLabel(job); +// +// TaskUtils.addLabelsMap(job.getParams(), labels); + + String code = parseExecutionCode(job); + + JobExecuteAction.Builder builder = JobExecuteAction.builder().setCreator(LINKIS_JOB_CREATOR.getValue(job.getJobProps())) + .addExecuteCode(code) + .setEngineTypeStr(parseAppConnEngineType(job.getEngineType(), job)) + .setRunTypeStr(parseRunType(job.getEngineType(), job.getRunType(), job)) + .setUser(job.getUser()) + .setParams(job.getParams()) + .setRuntimeParams(job.getRuntimeParams()); + if (job instanceof LinkisJob) { + Map source = new HashMap<>(); + source.putAll(((LinkisJob) job).getSource()); + builder = builder.setSource(source); + } + return builder.build(); + } + + @Override + public JobSubmitAction getSubmitAction(Job job) throws LinkisJobExecutionErrorException { + enrichParams(job); + + Map labels = prepareYarnLabel(job); + + TaskUtils.addLabelsMap(job.getParams(), labels); + + String code = parseExecutionCodeFor1X(job); + + EngineTypeLabel engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(parseAppConnEngineType(job.getEngineType(), job)); + + labels.put(LabelKeyConstant.ENGINE_TYPE_KEY, engineTypeLabel.getStringValue()); + labels.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, job.getUser() + "-" + LINKIS_JOB_CREATOR_1_X.getValue(job.getJobProps())); + labels.put(LabelKeyConstant.CODE_TYPE_KEY, parseRunType(job.getEngineType(), job.getRunType(), job)); + + + //是否复用引擎,不复用就为空 + if(!isReuseEngine(job.getParams())){ + labels.put("executeOnce", ""); + } + Map paramMapCopy = (HashMap) SerializationUtils.clone(new HashMap(job.getParams())); + JobSubmitAction.Builder builder = JobSubmitAction.builder() + .addExecuteCode(code) + .setUser(job.getUser()) + .addExecuteUser(job.getUser()) + .setParams(paramMapCopy) + .setLabels(labels) + .setRuntimeParams(job.getRuntimeParams()); + if (job instanceof LinkisJob) { + Map source = new HashMap<>(); + source.putAll(((LinkisJob) job).getSource()); + builder = builder.setSource(source); + } + // 将execute接口带来的额外variable参数,带进来 todo check + Map propMap = new HashMap<>(); + propMap.putAll(job.getJobProps()); + TaskUtils.addVariableMap(paramMapCopy, TaskUtils.getVariableMap(propMap)); + builder.setParams(paramMapCopy); + return builder.build(); + } + + /** + * 是否复用引擎,复用返回:true,不复用:false + * @param params + * @return + */ + public boolean isReuseEngine(Map params) { + if (params.get("configuration") != null) { + Map configurationMap = (Map) params.get("configuration"); + if (configurationMap.get("startup") != null) { + Map startupMap = (Map) configurationMap.get("startup"); + if (startupMap.get("ReuseEngine") != null) { + String reuseEngine = (String) startupMap.get("ReuseEngine"); + if (StringUtils.isNotBlank(reuseEngine) && "false".equalsIgnoreCase(reuseEngine.trim())) { + return false; + } + } + } + } + return true; + } + + //TODO + private String parseAppConnEngineType(String engineType, Job job) { + Map props = job.getJobProps(); + if (LinkisJobExecutionConfiguration.isLinkis1_X(props) && engineType.equalsIgnoreCase("appjoint")) { + return LinkisJobExecutionConfiguration.APPCONN; + } + return engineType; + } + + private String parseRunType(String engineType, String runType, Job job) { + Map props = job.getJobProps(); + if (LinkisJobExecutionConfiguration.isLinkis1_X(props) && engineType.equalsIgnoreCase("appjoint")) { + return LinkisJobExecutionConfiguration.APPCONN; + } else if (engineType.toLowerCase().contains("shell")) { + return "shell"; + } + + return runType; + } + + private void enrichParams(Job job) { + job.getRuntimeParams().put("nodeType", job.getRunType()); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/LinkisURLServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/LinkisURLServiceImpl.java new file mode 100644 index 000000000..80ac0469e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/LinkisURLServiceImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.service.impl; + +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.service.LinkisURLService; +import java.util.Map; + + +public class LinkisURLServiceImpl implements LinkisURLService { + + @Override + public String getLinkisURL(Job job) { + Map props = job.getJobProps(); + if(LinkisJobExecutionConfiguration.isLinkis1_X(props)){ + return LinkisJobExecutionConfiguration.LINKIS_URL_1_X.getValue(props); + } else { + return LinkisJobExecutionConfiguration.LINKIS_URL.getValue(props); + } + } + + @Override + public String getDefaultLinkisURL(Job job) { + return LinkisJobExecutionConfiguration.LINKIS_URL_1_X.getValue(job.getJobProps()); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisJobExecutionUtils.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisJobExecutionUtils.java new file mode 100644 index 000000000..e91264b72 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisJobExecutionUtils.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.utils; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; +import org.apache.commons.lang.StringUtils; + +import java.util.*; + +public class LinkisJobExecutionUtils { + + public final static Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + + public final static Gson gson1 = new GsonBuilder().disableHtmlEscaping().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + + public static BMLResource getBMLResourceByJson(String json){ + return gson.fromJson(json, BMLResource.class); + } + + public static ArrayList getResourceListByJson(String json){ + if(StringUtils.isEmpty(json)){ + return null; + } + return gson.fromJson(json,new TypeToken>() {}.getType()); + } + + public static Boolean isCommonAppConnJob(String engineType){ + return LinkisJobExecutionConfiguration.APPCONN.equalsIgnoreCase(engineType) || engineType.equalsIgnoreCase("appjoint"); + } + + public static final Integer IDX_FOR_LOG_TYPE_ALL = 3;// 0: Error 1: WARN 2:INFO 3: ALL +} + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisUjesClientUtils.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisUjesClientUtils.java new file mode 100644 index 000000000..0e17e5a64 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisUjesClientUtils.java @@ -0,0 +1,79 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.utils; + +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.linkis.common.exception.LinkisRetryException; +import org.apache.linkis.common.utils.DefaultRetryHandler; +import org.apache.linkis.common.utils.RetryHandler; +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.UJESClientImpl; + +import java.net.ConnectException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + + +public class LinkisUjesClientUtils { + + public static DWSClientConfig getClientConfig(String url, String user, String token, Map jobProps) { + return ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() + .addServerUrl(url) + .connectionTimeout(LinkisJobExecutionConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) + .discoveryEnabled(false).discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(false) + .maxConnectionSize(LinkisJobExecutionConfiguration.MAX_HTTP_CONNECTION_COUNT.getValue()) + .retryEnabled(false).readTimeout(LinkisJobExecutionConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) + .setAuthTokenKey(user).setAuthTokenValue(token))) + .setDWSVersion(LinkisJobExecutionConfiguration.LINKIS_API_VERSION.getValue(jobProps)).build(); + } + + public static DWSClientConfig getClientConfig1_X(String url, String user, String token, Map jobProps) { + RetryHandler retryHandler = new DefaultRetryHandler(); + retryHandler.addRetryException(LinkisRetryException.class); + retryHandler.addRetryException(ConnectTimeoutException.class); + retryHandler.addRetryException(ConnectException.class); + DWSClientConfigBuilder builder = (DWSClientConfigBuilder) DWSClientConfigBuilder.newBuilder() + .addServerUrl(url) + .connectionTimeout(LinkisJobExecutionConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) + .maxConnectionSize(LinkisJobExecutionConfiguration.MAX_HTTP_CONNECTION_COUNT.getValue()) + .readTimeout(LinkisJobExecutionConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) + .setAuthTokenKey(user).setAuthTokenValue(token) + .retryEnabled(true) + .setRetryHandler(retryHandler); + builder.setDWSVersion(LinkisJobExecutionConfiguration.LINKIS_API_VERSION.getValue(jobProps)).build(); + if(LinkisJobExecutionConfiguration.LINKIS_DISCOVERY_ENABLE.getValue()) { + builder.discoveryEnabled(true).discoveryFrequency(10, TimeUnit.MINUTES) + .loadbalancerEnabled(true); + } + return builder.build(); + } + + public static UJESClient getUJESClient(String url, String user, String token, Map jobProps) { + if (LinkisJobExecutionConfiguration.isLinkis1_X(jobProps)) { + return new UJESClientImpl(getClientConfig1_X(url, user, token, jobProps)); + } else { + return new UJESClientImpl(getClientConfig(url, user, token, jobProps)); + } + } +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/scala/org/apache/linkis/common/utils/DefaultRetryHandler.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/scala/org/apache/linkis/common/utils/DefaultRetryHandler.scala new file mode 100644 index 000000000..7dc6a46fa --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/scala/org/apache/linkis/common/utils/DefaultRetryHandler.scala @@ -0,0 +1,9 @@ +package org.apache.linkis.common.utils + +/** + * + * @date 2022-03-18 + * @author enjoyyin + * @since 1.1.0 + */ +class DefaultRetryHandler extends RetryHandler diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml new file mode 100644 index 000000000..6dfbdca06 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml @@ -0,0 +1,44 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-workflow-common + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-orchestrator-common + ${dss.version} + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlow.java new file mode 100644 index 000000000..fb930a11a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlow.java @@ -0,0 +1,256 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +public class DSSFlow implements Flow { + private Long id; + private String name; + private Boolean state; //0,1代表发布过和未发布过 + private String source; + private String description; + private Date createTime; + private String creator; + private Boolean isRootFlow; + private Integer rank; + private Long projectID; + private String linkedAppConnNames; + private String dssLabels; + private String flowEditLock;//工作流编辑锁 + /** + * 0disable 1 enable 0表示工作流从来没存过,发布的时候忽略 + */ + + private Boolean hasSaved; + private String uses; + private List children; + private String flowType; + private String resourceId; + + private String bmlVersion; + + public String getFlowJson() { + return flowJson; + } + + public void setFlowJson(String flowJson) { + this.flowJson = flowJson; + } + + private String flowJson; + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public Integer getRank() { + return rank; + } + + public void setRank(Integer rank) { + this.rank = rank; + } + + @Override + public Long getId() { + return id; + } + + @Override + public void setId(Long id) { + this.id = id; + } + + @Override + public String getName() { + return name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + + @Override + public String getDescription() { + return description; + } + + @Override + public void setDescription(String description) { + this.description = description; + } + + @Override + public List getChildren() { + return children; + } + + public void addChildren(DSSFlow children) { + if (this.children == null) { + this.children = new ArrayList<>(); + } + this.children.add(children); + } + + @Override + public void setChildren(List children) { + this.children = children.stream().map(f -> (DSSFlow) f).collect(Collectors.toList()); + } + + @Override + public String getFlowType() { + return flowType; + } + + @Override + public void setFlowType(String flowType) { + this.flowType = flowType; + } + + @Override + public Boolean getRootFlow() { + return isRootFlow; + } + + public void setRootFlow(Boolean rootFlow) { + isRootFlow = rootFlow; + } + + public Boolean getState() { + return state; + } + + public void setState(Boolean state) { + this.state = state; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Long getProjectID() { + return projectID; + } + + public void setProjectID(Long projectID) { + this.projectID = projectID; + } + + public Boolean getHasSaved() { + return hasSaved; + } + + public void setHasSaved(Boolean hasSaved) { + this.hasSaved = hasSaved; + } + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public String getLinkedAppConnNames() { + return linkedAppConnNames; + } + + public void setLinkedAppConnNames(String linkedAppConnNames) { + this.linkedAppConnNames = linkedAppConnNames; + } + + public String getDssLabels() { + return dssLabels; + } + + public void setDssLabels(String dssLabels) { + this.dssLabels = dssLabels; + } + + public String getFlowEditLock() { + return flowEditLock; + } + + public void setFlowEditLock(String flowEditLock) { + this.flowEditLock = flowEditLock; + } + + @Override + public String toString() { + return "DSSFlow{" + + "id=" + id + + ", name='" + name + '\'' + + ", state=" + state + + ", source='" + source + '\'' + + ", description='" + description + '\'' + + ", createTime=" + createTime + + ", creator='" + creator + '\'' + + ", isRootFlow=" + isRootFlow + + ", rank=" + rank + + ", projectID=" + projectID + + ", linkedAppConnNames='" + linkedAppConnNames + '\'' + + ", dssLabels='" + dssLabels + '\'' + + ", flowEditLock='" + flowEditLock + '\'' + + ", hasSaved=" + hasSaved + + ", uses='" + uses + '\'' + + ", children=" + children + + ", flowType='" + flowType + '\'' + + ", resourceId='" + resourceId + '\'' + + ", bmlVersion='" + bmlVersion + '\'' + + ", flowJson='" + flowJson + '\'' + + '}'; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlowRelation.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlowRelation.java new file mode 100644 index 000000000..2d5a35ac5 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlowRelation.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +public class DSSFlowRelation { + private Long id; + private Long flowID; + private Long parentFlowID; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getFlowID() { + return flowID; + } + + public void setFlowID(Long flowID) { + this.flowID = flowID; + } + + public Long getParentFlowID() { + return parentFlowID; + } + + public void setParentFlowID(Long parentFlowID) { + this.parentFlowID = parentFlowID; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSJsonFlow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSJsonFlow.java new file mode 100644 index 000000000..4635338c7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSJsonFlow.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +import java.util.List; +import java.util.stream.Collectors; + +public class DSSJsonFlow extends DSSFlow { + private String json; + + private List children; + + public String getJson() { + return json; + } + + public void setJson(String json) { + this.json = json; + } + + @Override + public void setChildren(List children) { + this.children = children.stream().map(f ->(DSSJsonFlow)f).collect(Collectors.toList()); + } + + @Override + public List getChildren() { + return children; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSPersonalFlow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSPersonalFlow.java new file mode 100644 index 000000000..6447b3795 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSPersonalFlow.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + + +public class DSSPersonalFlow extends DSSJsonFlow { + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSReleasedFlowVO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSReleasedFlowVO.java new file mode 100644 index 000000000..60c259b2d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSReleasedFlowVO.java @@ -0,0 +1,212 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +import java.util.Date; +import java.util.List; + +public class DSSReleasedFlowVO { + private Integer id; + private String workflowName; + private String projectName; + private String scheduleTime; + private String latestVersion; + private String lastScheduleStatus; + private String committer; + private String lastUpdater; + private Date lastUpdateTime; + private List accessUsers; + private Integer privModel; + private ScheduleInfo scheduleInfo; + private Long projectVersionId; + private Long projectId; + private FlowPriv flowPriv; + + public static class FlowPriv{ + private Integer privModel; + private List usernames; + + public Integer getPrivModel() { + return privModel; + } + + public void setPrivModel(Integer privModel) { + this.privModel = privModel; + } + + public List getUsernames() { + return usernames; + } + + public void setUsernames(List usernames) { + this.usernames = usernames; + } + } + + public static class ScheduleInfo{ + private String scheduleTime; + private String alarmUserEmails; + private String alarmLevel; + + public String getScheduleTime() { + return scheduleTime; + } + + public void setScheduleTime(String scheduleTime) { + this.scheduleTime = scheduleTime; + } + + public String getAlarmUserEmails() { + return alarmUserEmails; + } + + public void setAlarmUserEmails(String alarmUserEmails) { + this.alarmUserEmails = alarmUserEmails; + } + + public String getAlarmLevel() { + return alarmLevel; + } + + public void setAlarmLevel(String alarmLevel) { + this.alarmLevel = alarmLevel; + } + } + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getWorkflowName() { + return workflowName; + } + + public void setWorkflowName(String workflowName) { + this.workflowName = workflowName; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getScheduleTime() { + return scheduleTime; + } + + public void setScheduleTime(String scheduleTime) { + this.scheduleTime = scheduleTime; + } + + public String getLatestVersion() { + return latestVersion; + } + + public void setLatestVersion(String latestVersion) { + this.latestVersion = latestVersion; + } + + public String getLastScheduleStatus() { + return lastScheduleStatus; + } + + public void setLastScheduleStatus(String lastScheduleStatus) { + this.lastScheduleStatus = lastScheduleStatus; + } + + public String getCommitter() { + return committer; + } + + public void setCommitter(String committer) { + this.committer = committer; + } + + public String getLastUpdater() { + return lastUpdater; + } + + public void setLastUpdater(String lastUpdater) { + this.lastUpdater = lastUpdater; + } + + public Date getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(Date lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public List getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public Integer getPrivModel() { + return privModel; + } + + public void setPrivModel(Integer privModel) { + this.privModel = privModel; + } + + public ScheduleInfo getScheduleInfo() { + return scheduleInfo; + } + + public void setScheduleInfo(ScheduleInfo scheduleInfo) { + this.scheduleInfo = scheduleInfo; + } + + public Long getProjectVersionId() { + return projectVersionId; + } + + public void setProjectVersionId(Long projectVersionId) { + this.projectVersionId = projectVersionId; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public FlowPriv getFlowPriv() { + return flowPriv; + } + + public void setFlowPriv(FlowPriv flowPriv) { + this.flowPriv = flowPriv; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DWSFlowPublishHistory.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DWSFlowPublishHistory.java new file mode 100644 index 000000000..62a88ecd5 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DWSFlowPublishHistory.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + + +public class DWSFlowPublishHistory { +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DWSFlowVersion.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DWSFlowVersion.java new file mode 100644 index 000000000..80c665e00 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DWSFlowVersion.java @@ -0,0 +1,191 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +import java.util.Date; + + +public class DWSFlowVersion implements FlowVersion, Comparable { + private Long id; + private Long flowID; + private String source; + private String jsonPath; + private String comment; + private Date updateTime; + private String version; + private String bmlVersion; + private DWSFlowPublishHistory publishHistory; + private String json; + private String updator; + private Boolean isNotPublished; //true 未发过版,false已经过版 + private Long projectVersionID; + private Long oldFlowID; + /** + * 工作流是否被锁 + */ + private Boolean isFlowEditLockExist; + + public Boolean getFlowEditLockExist() { + return isFlowEditLockExist; + } + + public void setFlowEditLockExist(Boolean flowEditLockExist) { + isFlowEditLockExist = flowEditLockExist; + } + + /** + * 工作流编辑锁,打开自动获取,超时自动释放 + */ + private String flowEditLock; + + public String getFlowEditLock() { + return flowEditLock; + } + + public void setFlowEditLock(String flowEditLock) { + this.flowEditLock = flowEditLock; + } + + public Long getOldFlowID() { + return oldFlowID; + } + + public void setOldFlowID(Long oldFlowID) { + this.oldFlowID = oldFlowID; + } + + public Long getProjectVersionID() { + return projectVersionID; + } + + public void setProjectVersionID(Long projectVersionID) { + this.projectVersionID = projectVersionID; + } + + public Boolean getNotPublished() { + return isNotPublished; + } + + public void setNotPublished(Boolean notPublished) { + isNotPublished = notPublished; + } + + public String getUpdator() { + return updator; + } + + public void setUpdator(String updator) { + this.updator = updator; + } + + public String getJson() { + return json; + } + + public void setJson(String json) { + this.json = json; + } + + @Override + public String getVersion() { + return version; + } + + @Override + public void setVersion(String version) { + this.version = version; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getFlowID() { + return flowID; + } + + public void setFlowID(Long flowID) { + this.flowID = flowID; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getJsonPath() { + return jsonPath; + } + + public void setJsonPath(String jsonPath) { + this.jsonPath = jsonPath; + } + + @Override + public String getComment() { + return comment; + } + + @Override + public void setComment(String comment) { + this.comment = comment; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public DWSFlowPublishHistory getPublishHistory() { + return publishHistory; + } + + public void setPublishHistory(DWSFlowPublishHistory publishHistory) { + this.publishHistory = publishHistory; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + @Override + public int compareTo(DWSFlowVersion o) { + Integer v1 = Integer.valueOf(this.version.substring(1, version.length())); + Integer v2 = Integer.valueOf(o.version.substring(1, o.version.length())); + if (v1.equals(v2) && this.bmlVersion != null && o.bmlVersion != null) { + Integer bV1 = Integer.valueOf(this.bmlVersion.substring(1, bmlVersion.length())); + Integer bV2 = Integer.valueOf(o.bmlVersion.substring(1, bmlVersion.length())); + return bV2 - bV1; + } else { + return v2 - v1; + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FileFlowVersion.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FileFlowVersion.java new file mode 100644 index 000000000..ccd558d0c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FileFlowVersion.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +/* + FileFlowVersion 接口 + */ +public interface FileFlowVersion extends FlowVersion { + Object getFlowFile(); + + void setFlowFile(Object flowFile); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/Flow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/Flow.java new file mode 100644 index 000000000..ef8dc85c6 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/Flow.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestration; +import java.util.List; + +public interface Flow extends DSSOrchestration { + + void setId(Long id); + + void setName(String name); + + String getFlowType(); + + void setFlowType(String flowType); + + void setDescription(String description); + + Boolean getRootFlow(); + + List getChildren(); + + void setChildren(List children); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FlowVersion.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FlowVersion.java new file mode 100644 index 000000000..044f6d87d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FlowVersion.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; +/* + 顶级FlowVersion接口 +*/ +public interface FlowVersion { + String getVersion(); + + void setVersion(String version); + + String getComment(); + + void setComment(String comment); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FlowVersionForNodes.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FlowVersionForNodes.java new file mode 100644 index 000000000..8001e9a4c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/FlowVersionForNodes.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +import com.webank.wedatasphere.dss.common.entity.node.Node; + +import java.util.List; + +/* + FlowVersionForNodes 接口 + */ +public interface FlowVersionForNodes { + List getNodes(); + + void setNodes(List nodes); + + void addNode(Node node); + + void removeNode(Node node); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/JSONFlowVersion.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/JSONFlowVersion.java new file mode 100644 index 000000000..0ed6e4c21 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/JSONFlowVersion.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.entity; + +/* + JSONFlowVersion 接口 +*/ +public interface JSONFlowVersion extends FlowVersion { + Object getFlowJSON(); + + void setFlowJSON(Object flowJSON); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java new file mode 100644 index 000000000..53721e040 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.parser; + + +import com.webank.wedatasphere.dss.common.entity.Resource; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + + +public interface NodeParser { + String updateNodeResource(String nodeJson, List resources) throws IOException; + String updateNodeJobContent(String nodeJson, Map content) throws IOException; + String updateSubFlowID(String nodeJson, long subflowId) throws IOException; + String getNodeValue(String key, String nodeJson) throws IOException; + List getNodeResource(String nodeJson); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/WorkFlowParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/WorkFlowParser.java new file mode 100644 index 000000000..6a3a6820d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/WorkFlowParser.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.parser; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + + +public interface WorkFlowParser { + List getWorkFlowResources(String workFlowJson); + + List getWorkFlowNodes(String workFlowJson); + + List getWorkFlowEdges(String workFlowJson); + + List getWorkFlowNodesJson(String workFlowJson); + + String updateFlowJsonWithKey(String workFlowJson, String key, Object value) throws IOException; + + String updateFlowJsonWithMap(String workFlowJson, Map props) throws JsonProcessingException; + + String getValueWithKey(String workFlowJson, String key) throws IOException; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestCopyWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestCopyWorkflow.java new file mode 100644 index 000000000..0f0dfefbf --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestCopyWorkflow.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; + +public class RequestCopyWorkflow { + private String userName; + private Workspace workspace; + private Long rootFlowId; + private String contextIdStr; + private String projectName; + private String orcVersion; + private String description; + private List dssLabels; + + public RequestCopyWorkflow(String userName, + Workspace workspace, + Long rootFlowId, + String contextIdStr, + String projectName, + String orcVersion, + String description, + List dssLabels) { + + this.userName = userName; + this.workspace = workspace; + this.rootFlowId = rootFlowId; + this.contextIdStr = contextIdStr; + this.projectName = projectName; + this.orcVersion = orcVersion; + this.description = description; + this.dssLabels = dssLabels; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public Long getRootFlowId() { + return rootFlowId; + } + + public void setRootFlowId(Long rootFlowId) { + this.rootFlowId = rootFlowId; + } + + public String getContextIdStr() { + return contextIdStr; + } + + public void setContextIdStr(String contextIdStr) { + this.contextIdStr = contextIdStr; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getOrcVersion() { + return orcVersion; + } + + public void setOrcVersion(String orcVersion) { + this.orcVersion = orcVersion; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestCreateWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestCreateWorkflow.java new file mode 100644 index 000000000..d8b9d56bb --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestCreateWorkflow.java @@ -0,0 +1,148 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; + + +public class RequestCreateWorkflow { + + private String userName; + private Long projectId; + private String workflowName; + private String contextIDStr; + private String description; + private Long parentFlowID; + private String uses; + private List linkedAppConnNames; + private List dssLabels; + private String orcVersion; + private String schedulerAppConnName; + + public RequestCreateWorkflow(String userName, + Long projectId, + String workflowName, + String contextIDStr, + String description, + Long parentFlowID, + String uses, + List linkedAppConnNames, + List dssLabels, + String orcVersion, + String schedulerAppConnName) { + this.userName = userName; + this.projectId = projectId; + this.workflowName = workflowName; + this.contextIDStr = contextIDStr; + this.description = description; + this.parentFlowID = parentFlowID; + this.uses = uses; + this.linkedAppConnNames = linkedAppConnNames; + this.dssLabels =dssLabels; + this.orcVersion = orcVersion; + this.schedulerAppConnName = schedulerAppConnName; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getWorkflowName() { + return workflowName; + } + + public void setWorkflowName(String workflowName) { + this.workflowName = workflowName; + } + + public String getContextIDStr() { + return contextIDStr; + } + + public void setContextIDStr(String contextIDStr) { + this.contextIDStr = contextIDStr; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Long getParentFlowID() { + return parentFlowID; + } + + public void setParentFlowID(Long parentFlowID) { + this.parentFlowID = parentFlowID; + } + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public List getLinkedAppConnNames() { + return linkedAppConnNames; + } + + public void setLinkedAppConnNames(List linkedAppConnNames) { + this.linkedAppConnNames = linkedAppConnNames; + } + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public String getOrcVersion() { + return orcVersion; + } + + public void setOrcVersion(String orcVersion) { + this.orcVersion = orcVersion; + } + + public String getSchedulerAppConnName() { + return schedulerAppConnName; + } + + public void setSchedulerAppConnName(String schedulerAppConnName) { + this.schedulerAppConnName = schedulerAppConnName; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestImportWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestImportWorkflow.java new file mode 100644 index 000000000..5c5528875 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestImportWorkflow.java @@ -0,0 +1,130 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; + +public class RequestImportWorkflow { + private String userName; + private String resourceId; + private String bmlVersion; + private Long projectId; + private String projectName; + private String orcVersion; + private Workspace workspace; + private String contextId; + private List dssLabels; + + public String getContextId() { + return contextId; + } + + public void setContextId(String contextId) { + this.contextId = contextId; + } + + + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getOrcVersion() { + return orcVersion; + } + + public void setOrcVersion(String orcVersion) { + this.orcVersion = orcVersion; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public RequestImportWorkflow(String userName, + String resourceId, + String bmlVersion, + Long projectId, + String projectName, + String orcVersion, + Workspace workspace, + String contextId, + List dssLabels) { + this.userName = userName; + this.resourceId = resourceId; + this.bmlVersion = bmlVersion; + this.projectId = projectId; + this.projectName = projectName; + this.orcVersion = orcVersion; + this.workspace = workspace; + this.contextId = contextId; + this.dssLabels = dssLabels; + } + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseCopyWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseCopyWorkflow.java new file mode 100644 index 000000000..07b59a607 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseCopyWorkflow.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; + + +public class ResponseCopyWorkflow { + + private DSSFlow dssFlow; + + public ResponseCopyWorkflow(DSSFlow dssFlow) { + this.dssFlow = dssFlow; + } + + public DSSFlow getDssFlow() { + return dssFlow; + } + + public void setDssFlow(DSSFlow dssFlow) { + this.dssFlow = dssFlow; + } + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseCreateWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseCreateWorkflow.java new file mode 100644 index 000000000..9f4443cd3 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseCreateWorkflow.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; + + +public class ResponseCreateWorkflow { + + public DSSFlow getDssFlow() { + return dssFlow; + } + + public void setDssFlow(DSSFlow dssFlow) { + this.dssFlow = dssFlow; + } + + private DSSFlow dssFlow; +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseDeleteWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseDeleteWorkflow.java new file mode 100644 index 000000000..ad8273435 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseDeleteWorkflow.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + +import com.webank.wedatasphere.dss.common.protocol.JobStatus; + + +public class ResponseDeleteWorkflow { + private JobStatus jobStatus; + + public ResponseDeleteWorkflow(JobStatus jobStatus) { + this.jobStatus = jobStatus; + } + + public JobStatus getJobStatus() { + return jobStatus; + } + + public void setJobStatus(JobStatus jobStatus) { + this.jobStatus = jobStatus; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseImportWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseImportWorkflow.java new file mode 100644 index 000000000..9717ad525 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseImportWorkflow.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + +import com.webank.wedatasphere.dss.common.protocol.JobStatus; + +import java.util.List; +import java.util.Map; + + +public class ResponseImportWorkflow { + + + private JobStatus status; + /** + * the key of this map is workflow id, and value is json content. + */ + private Map workflows; + + public ResponseImportWorkflow(JobStatus status, Map workflows) { + this.status = status; + this.workflows = workflows; + } + + public JobStatus getStatus() { + return status; + } + + public void setStatus(JobStatus status) { + this.status = status; + } + + public Map getWorkflows() { + return workflows; + } + + public void setWorkflows(Map workflows) { + this.workflows = workflows; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseQueryWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseQueryWorkflow.java new file mode 100644 index 000000000..4ea22adad --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseQueryWorkflow.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; + + +public class ResponseQueryWorkflow { + + private DSSFlow dssFlow; + + public ResponseQueryWorkflow(DSSFlow dssFlow) { + this.dssFlow = dssFlow; + } + + + + public DSSFlow getDssFlow() { + return dssFlow; + } + + public void setDssFlow(DSSFlow dssFlow) { + this.dssFlow = dssFlow; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseUpdateWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseUpdateWorkflow.java new file mode 100644 index 000000000..3a87286e4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseUpdateWorkflow.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.common.protocol; + +import com.webank.wedatasphere.dss.common.protocol.JobStatus; + + +public class ResponseUpdateWorkflow { + private JobStatus jobStatus; + + public ResponseUpdateWorkflow(JobStatus jobStatus) { + this.jobStatus = jobStatus; + } + + public JobStatus getJobStatus() { + return jobStatus; + } + + public void setJobStatus(JobStatus jobStatus) { + this.jobStatus = jobStatus; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml new file mode 100644 index 000000000..19663c1a8 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml @@ -0,0 +1,43 @@ + + + + + + dss-workflow + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-workflow-conversion-standard + + + + com.webank.wedatasphere.dss + dss-workflow-sdk + ${dss.version} + + + com.webank.wedatasphere.dss + dss-orchestrator-conversion-standard + ${dss.version} + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/ProjectConversionIntegrationStandard.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/ProjectConversionIntegrationStandard.java new file mode 100644 index 000000000..72c9e0500 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/ProjectConversionIntegrationStandard.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.workflow.conversion; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.AbstractConversionIntegrationStandard; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.DSSToRelConversionService; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.RelToOrchestratorConversionService; +import com.webank.wedatasphere.dss.workflow.conversion.service.ProjectToRelConversionService; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public class ProjectConversionIntegrationStandard extends AbstractConversionIntegrationStandard { + + @Override + protected DSSToRelConversionService createDSSToRelConversionService() { + return new ProjectToRelConversionService(); + } + + @Override + protected RelToOrchestratorConversionService createRelToDSSConversionService() { + return null; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/WorkflowConversionIntegrationStandard.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/WorkflowConversionIntegrationStandard.java new file mode 100644 index 000000000..2402d4406 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/WorkflowConversionIntegrationStandard.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conversion; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.AbstractConversionIntegrationStandard; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.DSSToRelConversionService; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.RelToOrchestratorConversionService; +import com.webank.wedatasphere.dss.standard.app.development.service.RefQueryService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.workflow.conversion.service.WorkflowToRelConversionService; + + +public class WorkflowConversionIntegrationStandard extends AbstractConversionIntegrationStandard { + + @Override + protected DSSToRelConversionService createDSSToRelConversionService() { + return new WorkflowToRelConversionService(); + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ConvertedRel.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ConvertedRel.java new file mode 100644 index 000000000..8e6d4c8ad --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ConvertedRel.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conversion.entity; + + +public interface ConvertedRel extends PreConversionRel { + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/PreConversionRel.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/PreConversionRel.java new file mode 100644 index 000000000..29ebc034f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/PreConversionRel.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conversion.entity; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.DSSToRelConversionRequestRef; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import java.util.List; + + +public interface PreConversionRel { + + DSSToRelConversionRequestRef getDSSToRelConversionRequestRef(); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ProjectPreConversionRel.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ProjectPreConversionRel.java new file mode 100644 index 000000000..8c191e65d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ProjectPreConversionRel.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.workflow.conversion.entity; + +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; + +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public interface ProjectPreConversionRel extends PreConversionRel { + + List getWorkflows(); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ProjectPreConversionRelImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ProjectPreConversionRelImpl.java new file mode 100644 index 000000000..5bdc21076 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/ProjectPreConversionRelImpl.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conversion.entity; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.DSSToRelConversionRequestRef; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import java.util.List; + + +public class ProjectPreConversionRelImpl implements ProjectPreConversionRel { + + private List workflows; + private DSSToRelConversionRequestRef dssToRelConversionRequestRef; + + public void setWorkflows(List workflows) { + this.workflows = workflows; + } + + public void setDSSToRelConversionRequestRef( + DSSToRelConversionRequestRef dssToRelConversionRequestRef) { + this.dssToRelConversionRequestRef = dssToRelConversionRequestRef; + } + + @Override + public List getWorkflows() { + return workflows; + } + + @Override + public DSSToRelConversionRequestRef getDSSToRelConversionRequestRef() { + return dssToRelConversionRequestRef; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/WorkflowPreConversionRel.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/WorkflowPreConversionRel.java new file mode 100644 index 000000000..f827a5e15 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/WorkflowPreConversionRel.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.workflow.conversion.entity; + +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public interface WorkflowPreConversionRel extends PreConversionRel { + + Workflow getWorkflow(); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/WorkflowPreConversionRelImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/WorkflowPreConversionRelImpl.java new file mode 100644 index 000000000..1ad4a9b50 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/entity/WorkflowPreConversionRelImpl.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.workflow.conversion.entity; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.DSSToRelConversionRequestRef; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public class WorkflowPreConversionRelImpl implements WorkflowPreConversionRel { + + private DSSToRelConversionRequestRef dssToRelConversionRequestRef; + private Workflow workflow; + + @Override + public Workflow getWorkflow() { + return workflow; + } + + public void setWorkflow(Workflow workflow) { + this.workflow = workflow; + } + + @Override + public DSSToRelConversionRequestRef getDSSToRelConversionRequestRef() { + return dssToRelConversionRequestRef; + } + + public void setDSSToRelConversionRequestRef(DSSToRelConversionRequestRef dssToRelConversionRequestRef) { + this.dssToRelConversionRequestRef = dssToRelConversionRequestRef; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/AbstractDSSToRelConversionOperation.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/AbstractDSSToRelConversionOperation.java new file mode 100644 index 000000000..bf736831b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/AbstractDSSToRelConversionOperation.java @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.dss.workflow.conversion.operation; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.DSSToRelConversionOperation; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.DSSToRelConversionRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.utils.AppStandardClassUtils; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.core.WorkflowFactory; +import com.webank.wedatasphere.dss.workflow.core.json2flow.AbstractJsonToFlowParser; +import com.webank.wedatasphere.dss.workflow.core.json2flow.JsonToFlowParser; +import com.webank.wedatasphere.dss.workflow.core.json2flow.parser.WorkflowParser; + +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public abstract class AbstractDSSToRelConversionOperation> + extends DSSToRelConversionOperation { + + private List workflowToRelConverters; + private WorkflowToRelSynchronizer workflowToRelSynchronizer; + + @Override + protected String getAppConnName() { + return getConversionService().getAppStandard().getAppConnName(); + } + + @Override + public void init() { + super.init(); + workflowToRelConverters = AppStandardClassUtils.getInstance(getAppConnName()).getInstances(WorkflowToRelConverter.class).stream().sorted(Comparator.comparingInt(WorkflowToRelConverter::getOrder)) + .collect(Collectors.toList()); + workflowToRelSynchronizer = AppStandardClassUtils.getInstance(getAppConnName()).getInstanceOrWarn(WorkflowToRelSynchronizer.class); + workflowToRelSynchronizer.setDSSToRelConversionOperation(this); + JsonToFlowParser parser = WorkflowFactory.INSTANCE.getJsonToFlowParser(); + if (parser instanceof AbstractJsonToFlowParser) { + String packageName = WorkflowParser.class.getPackage().getName(); + List workflowParsers = AppStandardClassUtils.getInstance(getAppConnName()).getInstances(WorkflowParser.class).stream() + .filter(p -> !p.getClass().getName().startsWith(packageName) && + ((AbstractJsonToFlowParser) parser).getWorkflowParsers().stream().noneMatch(l -> l.getClass().getName() + .equals(p.getClass().getName()))).collect(Collectors.toList()); + ((AbstractJsonToFlowParser) parser).addWorkflowParsers(workflowParsers); + } + } + + protected abstract PreConversionRel getPreConversionRel(K ref); + + @Override + public ResponseRef convert(K ref) { + PreConversionRel preConversionRel = getPreConversionRel(ref); + ConvertedRel convertedRel = tryConvert(preConversionRel); + trySync(convertedRel); + return ResponseRef.newInternalBuilder().success(); + } + + protected ConvertedRel tryConvert(PreConversionRel rel) { + ConvertedRel convertedRel = null; + for (WorkflowToRelConverter workflowToRelConverter : workflowToRelConverters) { + if (convertedRel == null) { + convertedRel = workflowToRelConverter.convertToRel(rel); + } else { + convertedRel = workflowToRelConverter.convertToRel(convertedRel); + } + } + return convertedRel; + } + + protected void trySync(ConvertedRel convertedRel) { + workflowToRelSynchronizer.syncToRel(convertedRel); + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/ProjectToRelConversionOperation.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/ProjectToRelConversionOperation.java new file mode 100644 index 000000000..eabb50309 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/ProjectToRelConversionOperation.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conversion.operation; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.DSSToRelConversionRequestRef; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ProjectPreConversionRelImpl; +import com.webank.wedatasphere.dss.workflow.core.WorkflowFactory; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; + +import java.util.List; +import java.util.stream.Collectors; + + +public class ProjectToRelConversionOperation + extends AbstractDSSToRelConversionOperation { + + @Override + protected PreConversionRel getPreConversionRel(DSSToRelConversionRequestRef.ProjectToRelConversionRequestRefImpl ref) { + List workflows = ref.getDSSOrcList().stream().map(flow -> WorkflowFactory.INSTANCE.getJsonToFlowParser() + .parse((DSSFlow) flow)).collect(Collectors.toList()); + ProjectPreConversionRelImpl rel = new ProjectPreConversionRelImpl(); + rel.setWorkflows(workflows); + rel.setDSSToRelConversionRequestRef(ref); + return rel; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelConversionOperation.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelConversionOperation.java new file mode 100644 index 000000000..df4cfe343 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelConversionOperation.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.workflow.conversion.operation; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.DSSToRelConversionRequestRef; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.WorkflowPreConversionRelImpl; +import com.webank.wedatasphere.dss.workflow.core.WorkflowFactory; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public class WorkflowToRelConversionOperation + extends AbstractDSSToRelConversionOperation { + + @Override + protected PreConversionRel getPreConversionRel(DSSToRelConversionRequestRef.OrchestrationToRelConversionRequestRefImpl ref) { + Workflow workflow = WorkflowFactory.INSTANCE.getJsonToFlowParser().parse((DSSFlow) ref.getDSSOrchestration()); + WorkflowPreConversionRelImpl rel = new WorkflowPreConversionRelImpl(); + rel.setDSSToRelConversionRequestRef(ref); + rel.setWorkflow(workflow); + return rel; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelConverter.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelConverter.java new file mode 100644 index 000000000..7ef3f916f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelConverter.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conversion.operation; + +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; +import com.webank.wedatasphere.dss.workflow.conversion.entity.PreConversionRel; +import com.webank.wedatasphere.dss.workflow.core.order.Order; + + +public interface WorkflowToRelConverter extends Order { + + /** + * Try to convert DSS workflow to external app ref. + * Warn: The implementation should resolve the loop of children workflow's by yourself. + * @param rel The DSS workflow wait to convert + * @return Converted Rel + */ + ConvertedRel convertToRel(PreConversionRel rel); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelSynchronizer.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelSynchronizer.java new file mode 100644 index 000000000..f9dd792d8 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/operation/WorkflowToRelSynchronizer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conversion.operation; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.DSSToRelConversionOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.workflow.conversion.entity.ConvertedRel; + + +public interface WorkflowToRelSynchronizer { + + void setDSSToRelConversionOperation(DSSToRelConversionOperation dssToRelConversionOperation); + + void syncToRel(ConvertedRel convertedRel); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/service/ProjectToRelConversionService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/service/ProjectToRelConversionService.java new file mode 100644 index 000000000..2ef1ca3e3 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/service/ProjectToRelConversionService.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.workflow.conversion.service; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.DSSToRelConversionOperation; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.AbstractConversionService; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.DSSToRelConversionService; +import com.webank.wedatasphere.dss.workflow.conversion.operation.ProjectToRelConversionOperation; + +/** + * @author enjoyyin + * @date 2022-03-16 + * @since 0.5.0 + */ +public class ProjectToRelConversionService extends AbstractConversionService + implements DSSToRelConversionService { + @Override + public DSSToRelConversionOperation getDSSToRelConversionOperation() { + return getOrCreate(ProjectToRelConversionOperation::new, ProjectToRelConversionOperation.class); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/service/WorkflowToRelConversionService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/service/WorkflowToRelConversionService.java new file mode 100644 index 000000000..d1c81d430 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/src/main/java/com/webank/wedatasphere/dss/workflow/conversion/service/WorkflowToRelConversionService.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conversion.service; + +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.DSSToRelConversionOperation; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.AbstractConversionService; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.service.DSSToRelConversionService; +import com.webank.wedatasphere.dss.workflow.conversion.operation.WorkflowToRelConversionOperation; + + +public class WorkflowToRelConversionService extends AbstractConversionService + implements DSSToRelConversionService { + + @Override + public DSSToRelConversionOperation getDSSToRelConversionOperation() { + return getOrCreate(WorkflowToRelConversionOperation::new, WorkflowToRelConversionOperation.class); + } + + @Override + public boolean isConvertAllOrcs() { + return false; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml new file mode 100644 index 000000000..b049d7730 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml @@ -0,0 +1,53 @@ + + + + + + dss-workflow + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-workflow-sdk + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + com.webank.wedatasphere.dss + dss-workflow-common + ${dss.version} + + + org.springframework + spring-context + ${spring.version} + provided + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/WorkflowFactory.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/WorkflowFactory.java new file mode 100644 index 000000000..09ede336a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/WorkflowFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core; + +import com.webank.wedatasphere.dss.workflow.core.builder.WorkflowBuilder; +import com.webank.wedatasphere.dss.workflow.core.flow2json.FlowToJsonParser; +import com.webank.wedatasphere.dss.workflow.core.json2flow.JsonToFlowParser; + + +public interface WorkflowFactory { + + WorkflowBuilder createWorkflowBuilder(); + + JsonToFlowParser getJsonToFlowParser(); + + FlowToJsonParser getFlowToJsonParser(); + + WorkflowFactory INSTANCE = WorkflowFactoryImpl.createInstance(); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/WorkflowFactoryImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/WorkflowFactoryImpl.java new file mode 100644 index 000000000..d67f0a457 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/WorkflowFactoryImpl.java @@ -0,0 +1,76 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.workflow.core.builder.WorkflowBuilder; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowImpl; +import com.webank.wedatasphere.dss.workflow.core.flow2json.FlowToJsonParser; +import com.webank.wedatasphere.dss.workflow.core.json2flow.AbstractJsonToFlowParser; +import com.webank.wedatasphere.dss.workflow.core.json2flow.JsonToFlowParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class WorkflowFactoryImpl implements WorkflowFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowFactoryImpl.class); + + private JsonToFlowParser jsonToFlowParser; + + @Override + public WorkflowBuilder createWorkflowBuilder() { + return null; + } + + @Override + public JsonToFlowParser getJsonToFlowParser() { + if(jsonToFlowParser != null) { + return jsonToFlowParser; + } + synchronized (this) { + if(jsonToFlowParser == null) { + try { + jsonToFlowParser = ClassUtils.getInstance(JsonToFlowParser.class); + } catch (DSSErrorException e) { + jsonToFlowParser = new AbstractJsonToFlowParser() { + @Override + protected Workflow createWorkflow() { + return new WorkflowImpl(); + } + }; + } + jsonToFlowParser.init(); + LOGGER.info("JsonToFlowParser is {}.", jsonToFlowParser.getClass().getSimpleName()); + } + } + return jsonToFlowParser; + } + + @Override + public FlowToJsonParser getFlowToJsonParser() { + return null; + } + + static WorkflowFactory createInstance() { + WorkflowFactory workflowFactory = ClassUtils.getInstanceOrDefault(WorkflowFactory.class, new WorkflowFactoryImpl()); + LOGGER.info("WorkflowFactory is {}.", workflowFactory.getClass().getSimpleName()); + return workflowFactory; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/builder/WorkflowBuilder.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/builder/WorkflowBuilder.java new file mode 100644 index 000000000..3ed453b46 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/builder/WorkflowBuilder.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.builder; + +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import java.util.Map; + +/** + * More conveniently to build a workflow. + */ +public interface WorkflowBuilder { + + T create(String name, String description); + + T addWorkflowNode(WorkflowNode node); + + T addWorkflowNode(String parentNode, WorkflowNode node); + + T addChild(Workflow child); + + T addChild(String parentNode, Workflow child); + + T addFlowResource(Resource resource); + + T addFlowProperties(Map properties); + + Workflow build(); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/constant/WorkflowConstant.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/constant/WorkflowConstant.java new file mode 100644 index 000000000..a25e3d074 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/constant/WorkflowConstant.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.constant; + + +public class WorkflowConstant { + public final static String PROXY_USER = "user.to.proxy"; +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/Workflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/Workflow.java new file mode 100644 index 000000000..c630dfc7c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/Workflow.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.entity; + +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.workflow.common.entity.Flow; +import java.util.List; +import java.util.Map; + + +public interface Workflow extends Flow { + + Workflow getParentWorkflow(); + + List getWorkflowNodes(); + + List getWorkflowNodeEdges(); + + List getFlowResources(); + + List> getFlowProperties(); + + Long getCreateTime(); + + Long getUpdateTime(); + + String getUpdateUser(); + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowImpl.java new file mode 100644 index 000000000..fde37700d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowImpl.java @@ -0,0 +1,216 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.entity; + +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSJsonFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.Flow; +import org.apache.commons.collections.CollectionUtils; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class WorkflowImpl implements Workflow { + private Long id; + private String name; + private String description; + private String type; + private List children; + private Boolean isRootFlow; + private String userProxy; + private List workflowNodes; + private List workflowNodeEdges; + private List flowResources; + private List> flowProperties; + private DSSJsonFlow jsonFlow; + private Workflow parentWorkflow; + private Long createTime; + private Long updateTime; + private String updateUser; + + public DSSJsonFlow getJsonFlow() { + return jsonFlow; + } + + public void setJsonFlow(DSSJsonFlow jsonFlow) { + this.jsonFlow = jsonFlow; + } + + @Override + public Workflow getParentWorkflow() { + return parentWorkflow; + } + + public void setParentWorkflow(WorkflowImpl parentWorkflow) { + this.parentWorkflow = parentWorkflow; + } + + public String getUserProxy() { + return userProxy; + } + + public void setUserProxy(String userProxy) { + this.userProxy = userProxy; + } + + @Override + public List getWorkflowNodes() { + return workflowNodes; + } + + public void setWorkflowNodes(List workflowNodes) { + this.workflowNodes = workflowNodes; + } + + @Override + public List getWorkflowNodeEdges() { + return workflowNodeEdges; + } + + public void setWorkflowNodeEdges(List workflowNodeEdges) { + this.workflowNodeEdges = workflowNodeEdges; + } + + @Override + public List getFlowResources() { + return flowResources; + } + + public void setFlowResources(List flowResources) { + this.flowResources = flowResources; + } + + @Override + public List> getFlowProperties() { + return flowProperties; + } + + public void setFlowProperties(List> flowProperties) { + try{ + if(CollectionUtils.isNotEmpty(flowProperties)){ + flowProperties.forEach(a->{ + String key = "user.to.proxy"; + if(a.containsKey(key)){ + this.setUpdateUser((String)a.get(key)); + } + }); + } + }catch (Exception e){ + e.fillInStackTrace(); + } + this.flowProperties = flowProperties; + } + + @Override + public Long getId() { + return this.id; + } + + @Override + public void setId(Long id) { + this.id = id; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public String getFlowType() { + return this.type; + } + + @Override + public void setFlowType(String flowType) { + this.type = flowType; + + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public void setDescription(String description) { + this.description = description; + } + + @Override + public Boolean getRootFlow() { + return this.isRootFlow; + } + + public void setRootFlow(Boolean rootFlow) { + isRootFlow = rootFlow; + } + + @Override + public List getChildren() { + return this.children; + } + + @Override + public void setChildren(List children) { + if(null!=children) { + this.children = children.stream().map(f -> (WorkflowImpl) f).collect(Collectors.toList()); + } + } + + @Override + public Long getCreateTime() { + return createTime; + } + + public void setCreateTime(Long createTime) { + this.createTime = createTime; + } + + @Override + public Long getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Long updateTime) { + this.updateTime = updateTime; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @Override + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNode.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNode.java new file mode 100644 index 000000000..a4908dedf --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNode.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.entity; + + +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.entity.node.Node; + +public interface WorkflowNode extends Node { + + DSSNode getDSSNode(); + + void setDSSNode(DSSNode dwsNode); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeEdge.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeEdge.java new file mode 100644 index 000000000..5669014f1 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeEdge.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.entity; + +import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; + +public interface WorkflowNodeEdge { + DSSEdge getDSSEdge(); + + void setDSSEdge(DSSEdge dssEdge); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeEdgeImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeEdgeImpl.java new file mode 100644 index 000000000..40f726eaf --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeEdgeImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.entity; + +import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; + + +public class WorkflowNodeEdgeImpl implements WorkflowNodeEdge { + private DSSEdge dssEdge; + + @Override + public DSSEdge getDSSEdge() { + return dssEdge; + } + + @Override + public void setDSSEdge(DSSEdge dssEdge) { + this.dssEdge = dssEdge; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeImpl.java new file mode 100644 index 000000000..5ae20ece0 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowNodeImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.entity; + +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; + +import java.util.List; + + +public class WorkflowNodeImpl implements WorkflowNode { + + private DSSNode dssNode; + + @Override + public DSSNode getDSSNode() { + return this.dssNode; + } + + @Override + public void setDSSNode(DSSNode dssNode) { + this.dssNode = dssNode; + } + + @Override + public String getId() { + return dssNode.getId(); + } + + @Override + public void setId(String id) { + dssNode.setId(id); + } + + @Override + public String getNodeType() { + return dssNode.getNodeType(); + } + + @Override + public void setNodeType(String nodeType) { + dssNode.setNodeType(nodeType); + } + + @Override + public String getName() { + return dssNode.getName(); + } + + @Override + public void setName(String name) { + dssNode.setName(name); + } + + @Override + public void addDependency(String nodeName) { + dssNode.addDependency(nodeName); + } + + @Override + public void setDependency(List dependency) { + dssNode.setDependency(dependency); + } + + @Override + public void removeDependency(String nodeName) { + dssNode.removeDependency(nodeName); + } + + @Override + public List getDependencys() { + return dssNode.getDependencys(); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowWithContextImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowWithContextImpl.java new file mode 100644 index 000000000..830125d44 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/entity/WorkflowWithContextImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.entity; + + +public class WorkflowWithContextImpl extends WorkflowImpl { + + private String contextID; + + public String getContextID() { + return contextID; + } + + public void setContextID(String contextID) { + this.contextID = contextID; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/flow2json/FlowToJsonParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/flow2json/FlowToJsonParser.java new file mode 100644 index 000000000..3b9185c14 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/flow2json/FlowToJsonParser.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.flow2json; + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSJsonFlow; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; + +/** + * Used to parse a workflow to DSSJsonFlow, it is useful when we want to use + * Workflow java SDK to create workflows and store it to db. + */ +public interface FlowToJsonParser { + + /** + * TODO wait for implementation. + * @param workflow + * @return + */ + DSSJsonFlow parse(Workflow workflow); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/AbstractJsonToFlowParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/AbstractJsonToFlowParser.java new file mode 100644 index 000000000..ec3a9cd1f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/AbstractJsonToFlowParser.java @@ -0,0 +1,104 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowImpl; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowWithContextImpl; +import com.webank.wedatasphere.dss.workflow.core.json2flow.parser.WorkflowParser; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import org.springframework.beans.BeanUtils; + + +public abstract class AbstractJsonToFlowParser implements JsonToFlowParser { + + private List workflowParsers; + + public List getWorkflowParsers() { + return workflowParsers; + } + + public synchronized void addWorkflowParsers(List workflowParsers) { + if(workflowParsers == null || workflowParsers.isEmpty()) { + return; + } + workflowParsers.forEach(WorkflowParser::init); + if(this.workflowParsers == null) { + this.workflowParsers = new ArrayList<>(); + } + this.workflowParsers.addAll(workflowParsers); + this.workflowParsers = this.workflowParsers.stream().sorted(Comparator.comparingInt(WorkflowParser::getOrder)).collect(Collectors.toList()); + } + + @Override + public void init() { + if(workflowParsers == null) { + workflowParsers = ClassUtils.getInstances(WorkflowParser.class); + } else { + workflowParsers.addAll(ClassUtils.getInstances(WorkflowParser.class)); + } + workflowParsers = workflowParsers.stream().sorted(Comparator.comparingInt(WorkflowParser::getOrder)).collect(Collectors.toList()); + workflowParsers.forEach(WorkflowParser::init); + } + + @Override + public Workflow parse(DSSFlow dssFlow) { + Workflow workflow = createWorkflow(); + BeanUtils.copyProperties(dssFlow, workflow, "children"); + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(dssFlow.getFlowJson()).getAsJsonObject(); + for(WorkflowParser workflowParser : workflowParsers) { + workflow = workflowParser.parse(jsonObject, workflow); + } + if(dssFlow.getChildren() != null) { + Workflow parentWorkflow = workflow; + List children = dssFlow.getChildren().stream().map(childFlow -> { + Workflow childWorkflow = parse(childFlow); + if(parentWorkflow instanceof WorkflowImpl) { + ((WorkflowImpl) childWorkflow).setParentWorkflow((WorkflowImpl) parentWorkflow); + } + return childWorkflow; + }).collect(Collectors.toList()); + workflow.setChildren(children); + } + syncJsonField(workflow,jsonObject); + return workflow; + } + + public void syncJsonField(Workflow workflow,JsonObject jsonObject){ + try{ + if(!jsonObject.has("updateTime")){ + return; + } + if(workflow instanceof WorkflowImpl) { + ((WorkflowImpl) workflow).setUpdateTime(jsonObject.get("updateTime").getAsLong()); + }else if(workflow instanceof WorkflowWithContextImpl) { + ((WorkflowWithContextImpl) workflow).setUpdateTime(jsonObject.get("updateTime").getAsLong()); + } + }catch (Exception e){ + e.fillInStackTrace(); + } + } + protected abstract Workflow createWorkflow(); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/JsonToFlowParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/JsonToFlowParser.java new file mode 100644 index 000000000..c21078ec8 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/JsonToFlowParser.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow; + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; + + +public interface JsonToFlowParser { + + void init(); + + Workflow parse(DSSFlow dssFlow); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowContextParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowContextParser.java new file mode 100644 index 000000000..2e9a15665 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowContextParser.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow.parser; + +import com.google.gson.JsonObject; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowWithContextImpl; +import org.springframework.beans.BeanUtils; + + +public class WorkflowContextParser implements WorkflowParser { + + @Override + public Workflow parse(JsonObject flowJson, Workflow workflow) { + if (flowJson.has("contextID")) { + String contextID = flowJson.get("contextID").getAsString(); + WorkflowWithContextImpl workflowWithContext = new WorkflowWithContextImpl(); + BeanUtils.copyProperties(workflow, workflowWithContext); + workflowWithContext.setContextID(contextID); + return workflowWithContext; + } else { + return workflow; + } + } + + @Override + public int getOrder() { + return 10; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowDAGParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowDAGParser.java new file mode 100644 index 000000000..21a4c8ef8 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowDAGParser.java @@ -0,0 +1,102 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow.parser; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; +import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; +import com.webank.wedatasphere.dss.common.entity.node.DSSEdgeDefault; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowImpl; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNodeEdge; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNodeEdgeImpl; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + + +public class WorkflowDAGParser implements WorkflowParser { + + private List workflowNodeParsers; + + public synchronized void addWorkflowNodeParser(WorkflowNodeParser workflowNodeParser) { + if(workflowNodeParsers == null) { + workflowNodeParsers = new ArrayList<>(); + } + this.workflowNodeParsers.add(workflowNodeParser); + } + + protected List getWorkflowNodeParsers() { + return workflowNodeParsers; + } + + @Override + public void init() { + if(workflowNodeParsers == null) { + workflowNodeParsers = ClassUtils.getInstances(WorkflowNodeParser.class); + } else { + workflowNodeParsers.addAll(ClassUtils.getInstances(WorkflowNodeParser.class)); + } + } + + @Override + public Workflow parse(JsonObject flowJson, Workflow workflow) { + JsonArray nodeJsonArray = flowJson.getAsJsonArray("nodes"); + Gson gson = DSSCommonUtils.COMMON_GSON; + List dwsNodes = gson.fromJson(nodeJsonArray, new TypeToken>() { + }.getType()); + List workflowNodeList = new ArrayList<>(); + List workflowNodeEdgeList = new ArrayList<>(); + if (null != dwsNodes) { + for (DSSNode dwsNode : dwsNodes) { + Optional firstNodeParser = workflowNodeParsers.stream() + .filter(p -> p.ifNodeCanParse(dwsNode)) + .min((p1, p2) -> p2.getOrder() - p1.getOrder()); + WorkflowNode workflowNode = firstNodeParser.orElseThrow(() -> new IllegalArgumentException("NodeParser个数应该大于0")).parseNode(dwsNode); + workflowNodeList.add(workflowNode); + } + } + JsonArray edgeJsonArray = flowJson.getAsJsonArray("edges"); + List dwsEdges = gson.fromJson(edgeJsonArray, new TypeToken>() { + }.getType()); + if (dwsEdges != null){ + for (DSSEdge dwsEdge : dwsEdges) { + WorkflowNodeEdge workflowNodeEdge = new WorkflowNodeEdgeImpl(); + workflowNodeEdge.setDSSEdge(dwsEdge); + workflowNodeEdgeList.add(workflowNodeEdge); + } + } + if(workflow instanceof WorkflowImpl) { + WorkflowImpl workflow1 = (WorkflowImpl) workflow; + workflow1.setWorkflowNodeEdges(workflowNodeEdgeList); + workflow1.setWorkflowNodes(workflowNodeList); + } + return workflow; + } + + @Override + public int getOrder() { + return 0; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowDependencyParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowDependencyParser.java new file mode 100644 index 000000000..0ce6158f9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowDependencyParser.java @@ -0,0 +1,93 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow.parser; + +import com.google.gson.JsonObject; +import com.webank.wedatasphere.dss.workflow.core.constant.WorkflowConstant; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowImpl; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNodeEdge; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang.StringUtils; + + +public class WorkflowDependencyParser implements WorkflowParser { + + @Override + public Workflow parse(JsonObject flowJson, Workflow workflow) { + if(!(workflow instanceof WorkflowImpl)) { + return workflow; + } + //1.设置依赖,对于多个flowTuning,已经设置的就跳过 + WorkflowImpl realWorkflow = (WorkflowImpl) workflow; + workflow.getWorkflowNodes() + .forEach(node -> setDependencies(node, workflow.getWorkflowNodes(), workflow.getWorkflowNodeEdges())); + //2.设置usreproxy + setProxyUser(realWorkflow); + return workflow; + } + + private void setDependencies(WorkflowNode node, List workflowNodes, List flowEdges){ + //设置过的flow不在进行设置和解析 + if(node.getDependencys()!= null && !node.getDependencys().isEmpty()) { + return; + } + List dependencies = resolveDependencies(node, workflowNodes ,flowEdges); + dependencies.forEach(node::addDependency); + } + + private List resolveDependencies(WorkflowNode node, List workflowNodes, List flowEdges) { + List dependencies = new ArrayList<>(); + flowEdges.forEach(edge -> { + if (edge.getDSSEdge().getTarget().equals(node.getId())) { + dependencies.add(workflowNodes.stream().filter(n ->edge.getDSSEdge().getSource().equals(n.getId())).findFirst().get().getName()); + } + }); + + return dependencies; + } + + private String getProxyUser(Workflow workflow) { + if(workflow.getFlowProperties() == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + workflow.getFlowProperties().forEach( map -> { + Object value = map.get(WorkflowConstant.PROXY_USER); + if(value != null && StringUtils.isNotBlank(value.toString())) { + sb.append(value.toString()); + } + }); + return sb.toString(); + } + + private void setProxyUser(WorkflowImpl workflow) { + String proxyUser = getProxyUser(workflow); + if(StringUtils.isNotBlank(proxyUser)) { + workflow.getWorkflowNodes().forEach(node -> node.getDSSNode().setUserProxy(proxyUser)); + workflow.setUserProxy(proxyUser); + } + } + + @Override + public int getOrder() { + return 20; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowNodeParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowNodeParser.java new file mode 100644 index 000000000..c74816e1f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowNodeParser.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow.parser; + +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import com.webank.wedatasphere.dss.workflow.core.order.Order; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; + +public interface WorkflowNodeParser extends Order { + + WorkflowNode parseNode(DSSNode dwsNode); + + Boolean ifNodeCanParse(DSSNode dwsNode); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowNodeParserImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowNodeParserImpl.java new file mode 100644 index 000000000..b8b46e086 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowNodeParserImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow.parser; + +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNodeImpl; + + +public class WorkflowNodeParserImpl implements WorkflowNodeParser { + @Override + public WorkflowNode parseNode(DSSNode dwsNode) { + WorkflowNodeImpl node = new WorkflowNodeImpl(); + node.setDSSNode(dwsNode); + return node; + } + + @Override + public Boolean ifNodeCanParse(DSSNode dwsNode) { + return true; + } + + @Override + public int getOrder() { + return 0; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowParser.java new file mode 100644 index 000000000..0d8f4ab25 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowParser.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow.parser; + +import com.google.gson.JsonObject; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.order.Order; + +public interface WorkflowParser extends Order { + + default void init() {} + + Workflow parse(JsonObject flowJson, Workflow workflow); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowPropsParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowPropsParser.java new file mode 100644 index 000000000..5c47eeec9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowPropsParser.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.json2flow.parser; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowImpl; +import java.util.List; +import java.util.Map; + + +public class WorkflowPropsParser implements WorkflowParser { + + @Override + public Workflow parse(JsonObject flowJson, Workflow workflow) { + JsonArray proJsonArray = flowJson.getAsJsonArray("props"); + List> props = DSSCommonUtils.COMMON_GSON.fromJson(proJsonArray, new TypeToken>>() { + }.getType()); + JsonArray resourcesJsonArray = flowJson.getAsJsonArray("resources"); + List resources = DSSCommonUtils.COMMON_GSON.fromJson(resourcesJsonArray, new TypeToken>() { + }.getType()); + if(workflow instanceof WorkflowImpl) { + WorkflowImpl workflow1 = (WorkflowImpl) workflow; + workflow1.setFlowResources(resources); + workflow1.setFlowProperties(props); + } + return workflow; + } + + @Override + public int getOrder() { + return 8; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/order/Order.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/order/Order.java new file mode 100644 index 000000000..791e66cc0 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/order/Order.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.core.order; + +/** + * order接口用于标识parser和tunning,hook的调用顺序 + */ +public interface Order { + + int getOrder(); + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml new file mode 100644 index 000000000..3d3190376 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml @@ -0,0 +1,214 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-workflow-server + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + linkis-bml-client + + + gson + com.google.code.gson + + + linkis-common + org.apache.linkis + + + commons-beanutils + commons-beanutils + + + commons-collections + commons-collections + + + commons-logging + commons-logging + + + json4s-jackson_2.11 + org.json4s + + + org.apache.linkis + ${linkis.version} + + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + + + + com.webank.wedatasphere.dss + dss-contextservice + ${dss.version} + + + linkis-common + org.apache.linkis + + + commons-text + org.apache.commons + + + + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + + + com.webank.wedatasphere.dss + dss-standard-common + ${dss.version} + + + com.webank.wedatasphere.dss + dss-scheduler-appconn + ${dss.version} + compile + + + commons-collections + commons-collections + + + jackson-databind + com.fasterxml.jackson.core + + + + + + com.webank.wedatasphere.dss + dss-appconn-manager-client + ${dss.version} + + + com.webank.wedatasphere.dss + dss-orchestrator-common + ${dss.version} + compile + + + com.webank.wedatasphere.dss + dss-sender-service + ${dss.version} + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/assembly/distribution.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/assembly/distribution.xml new file mode 100644 index 000000000..abcfce7c6 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/assembly/distribution.xml @@ -0,0 +1,43 @@ + + + + dss-workflow-server + + dir + + true + dss-workflow-server + + + + + + lib + true + true + false + true + true + + + + + + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java new file mode 100644 index 000000000..f42f094c7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java @@ -0,0 +1,296 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestConvertOrchestrations; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.operation.DSSToRelConversionOperation; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.DSSToRelConversionRequestRef; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.OrchestrationToRelConversionRequestRef; +import com.webank.wedatasphere.dss.orchestrator.converter.standard.ref.ProjectToRelConversionRequestRef; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.utils.RequestRefUtils; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam; +import com.webank.wedatasphere.dss.workflow.io.export.WorkFlowExportService; +import com.webank.wedatasphere.dss.workflow.io.input.MetaInputService; +import com.webank.wedatasphere.dss.workflow.io.input.WorkFlowInputService; +import com.webank.wedatasphere.dss.workflow.service.BMLService; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.linkis.protocol.util.ImmutablePair; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.DEFAULT_SCHEDULER_APP_CONN; + +@Component +public class DefaultWorkFlowManager implements WorkFlowManager { + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private DSSFlowService flowService; + + @Autowired + private WorkFlowInputService workFlowInputService; + + @Autowired + private WorkFlowExportService workFlowExportService; + + @Autowired + private BMLService bmlService; + + @Autowired + private MetaInputService metaInputService; + @Autowired + private WorkFlowParser workFlowParser; + + @Override + public DSSFlow createWorkflow(String userName, + Long projectId, + String flowName, + String contextIdStr, + String description, + Long parentFlowId, + String uses, + List linkedAppConnNames, + List dssLabels, + String orcVersion, + String schedulerAppConn) throws DSSErrorException, JsonProcessingException { + DSSFlow dssFlow = new DSSFlow(); + dssFlow.setName(flowName); + dssFlow.setProjectID(projectId); + dssFlow.setDescription(description); + dssFlow.setCreator(userName); + dssFlow.setCreateTime(new Date()); + dssFlow.setState(false); + dssFlow.setSource("create by user"); + dssFlow.setUses(uses); + + dssFlow.setLinkedAppConnNames(String.join(",", linkedAppConnNames)); + Map dssLabelList = new HashMap<>(1); + if (null != dssLabels) { + dssLabels.stream().map(DSSLabel::getValue).forEach(a->{ + dssLabelList.put(EnvDSSLabel.DSS_ENV_LABEL_KEY, a.get(EnvDSSLabel.DSS_ENV_LABEL_KEY)); + }); + } else { + dssLabelList.put(EnvDSSLabel.DSS_ENV_LABEL_KEY, DSSCommonUtils.ENV_LABEL_VALUE_DEV); + } + dssFlow.setDssLabels(BDPJettyServerHelper.jacksonJson().writeValueAsString(dssLabelList)); + //-1是rpc传递消息不支持null,所有父工作流都是-1。 + if (parentFlowId == null || parentFlowId == -1) { + dssFlow.setRootFlow(true); + dssFlow.setRank(0); + dssFlow.setHasSaved(true); + dssFlow = flowService.addFlow(dssFlow, contextIdStr, orcVersion, schedulerAppConn); + } else { + dssFlow.setRootFlow(false); + DSSFlow parentFlow = flowService.getFlowByID(parentFlowId); + // TODO 并发问题考虑 for update,由于加了工作流编辑锁,暂时可忽略 + dssFlow.setRank(parentFlow.getRank() + 1); + dssFlow.setHasSaved(true); + dssFlow.setProjectID(parentFlow.getProjectID()); + dssFlow = flowService.addSubFlow(dssFlow, parentFlowId, contextIdStr, orcVersion, schedulerAppConn); + } + return dssFlow; + } + + @Override + public DSSFlow copyRootFlowWithSubFlows(String userName, Long rootFlowId, Workspace workspace, + String projectName, String contextIdStr, String orcVersion, + String description, List dssLabels) throws DSSErrorException, IOException { + return flowService.copyRootFlow(rootFlowId, userName, workspace, projectName, + orcVersion, contextIdStr, description, dssLabels); + } + + @Override + public DSSFlow queryWorkflow(String userName, Long rootFlowId) { + return flowService.getFlowWithJsonAndSubFlowsByID(rootFlowId); + } + + @Override + public void updateWorkflow(String userName, Long flowId, String flowName, String description, String uses) throws DSSErrorException { + DSSFlow dssFlow = new DSSFlow(); + dssFlow.setId(flowId); + dssFlow.setName(flowName); + dssFlow.setDescription(description); + dssFlow.setUses(uses); + flowService.updateFlowBaseInfo(dssFlow); + } + + @Override + public void deleteWorkflow(String userName, Long flowId) throws DSSErrorException { + DSSFlow dssFlow = flowService.getFlowByID(flowId); + if (dssFlow.getCreator().equals(userName)) { + flowService.batchDeleteFlow(Collections.singletonList(flowId)); + } else { + throw new DSSErrorException(100088, "Workflow can not be deleted unless the owner."); + } + } + + @Override + public Map exportWorkflow(String userName, Long flowId, Long dssProjectId, + String projectName, Workspace workspace, + List dssLabels) throws Exception { + DSSFlow dssFlow = flowService.getFlowByID(flowId); + String exportPath = workFlowExportService.exportFlowInfo(dssProjectId, projectName, flowId, userName, workspace,dssLabels); + InputStream inputStream = bmlService.readLocalResourceFile(userName, exportPath); + return bmlService.upload(userName, inputStream, dssFlow.getName() + ".export", projectName); + } + + @Override + public List importWorkflow(String userName, + String resourceId, + String bmlVersion, + DSSFlowImportParam dssFlowImportParam, + List dssLabels) throws DSSErrorException, IOException { + + //todo download workflow bml file contains flowInfo and flowRelationInfo + String inputZipPath = IoUtils.generateIOPath(userName, dssFlowImportParam.getProjectName(), dssFlowImportParam.getProjectName() + ".zip"); + bmlService.downloadToLocalPath(userName, resourceId, bmlVersion, inputZipPath); + String inputPath = ZipHelper.unzip(inputZipPath); + //导入工作流数据库信息 + List dssFlows = metaInputService.inputFlow(inputPath); + //导入工作流关系信息 + List dwsFlowRelations = metaInputService.inputFlowRelation(inputPath); + //兼容0.x导入 + if (CollectionUtils.isEmpty(dwsFlowRelations)) { + dwsFlowRelations = metaInputService.inputFlowRelation_for_0_x(inputPath); + } + + List dwsFlowRelationList = workFlowInputService.persistenceFlow(dssFlowImportParam.getProjectID(), + dssFlowImportParam.getUserName(), + dssFlows, + dwsFlowRelations); + List rootFlows = dwsFlowRelationList.stream().filter(DSSFlow::getRootFlow).collect(Collectors.toList()); + for (DSSFlow rootFlow : rootFlows) { + workFlowInputService.inputWorkFlow(dssFlowImportParam.getUserName(), + rootFlow, + dssFlowImportParam.getProjectName(), + inputPath, null, dssFlowImportParam.getWorkspace(), dssFlowImportParam.getOrcVersion(), + dssFlowImportParam.getContextId(), dssLabels); + } + return rootFlows; + } + + @Override + public ResponseOperateOrchestrator convertWorkflow(RequestConvertOrchestrations requestConversionWorkflow) { + if(requestConversionWorkflow.getOrchestrationIdMap() == null || requestConversionWorkflow.getOrchestrationIdMap().isEmpty()) { + logger.info("the project {} has no workflow, the conversion by user {} is ignored.", requestConversionWorkflow.getProject().getName(), + requestConversionWorkflow.getUserName()); + return ResponseOperateOrchestrator.failed("No workflow found, publish is ignored."); + } + logger.info("user {} try to convert workflowId(s) {} in project {} to SchedulerAppConn(s).", requestConversionWorkflow.getUserName(), + requestConversionWorkflow.getOrchestrationIdMap().keySet(), requestConversionWorkflow.getProject().getName()); + //TODO try to optimize it by select db in batch. + List> flowInfos = requestConversionWorkflow.getOrchestrationIdMap().entrySet() + .stream().map(entry -> new ImmutablePair<>(flowService.getFlowWithJsonAndSubFlowsByID(entry.getKey()), entry.getValue())) + .collect(Collectors.toList()); + List flows = flowInfos.stream().map(ImmutablePair::getKey).collect(Collectors.toList()); + // 区分各个工作流所归属的调度系统 + List responseList = new ArrayList<>(); + flows.stream().map(DSSExceptionUtils.map(flow -> { + String schedulerAppConnName = workFlowParser.getValueWithKey(flow.getFlowJson(), DSSWorkFlowConstant.SCHEDULER_APP_CONN_NAME); + if(StringUtils.isBlank(schedulerAppConnName)) { + // 向下兼容老版本 + schedulerAppConnName = DEFAULT_SCHEDULER_APP_CONN.getValue(); + } + return new ImmutablePair<>(schedulerAppConnName, flow); + })).collect(Collectors.groupingBy(ImmutablePair::getKey)).forEach((appConnName, pairList) -> { + List selectedFlows = pairList.stream().map(ImmutablePair::getValue).collect(Collectors.toList()); + ResponseOperateOrchestrator response = convert(requestConversionWorkflow, appConnName, selectedFlows, flowInfos); + responseList.add(response); + }); + List failedResponseList = responseList.stream().filter(ResponseOperateOrchestrator::isFailed).collect(Collectors.toList()); + if(!failedResponseList.isEmpty()) { + return ResponseOperateOrchestrator.failed("由于该 Project 包含指向多个调度系统的工作流,发布过程中有一部分失败了。失败部分如下:" + + failedResponseList.stream().map(ResponseOperateOrchestrator::getMessage).collect(Collectors.joining("; "))); + } else { + return responseList.get(0); + } + } + + private ResponseOperateOrchestrator convert(RequestConvertOrchestrations requestConversionWorkflow, + String schedulerAppConnName, + List flows, + List> flowInfos) { + String convertFlowStr = flows.stream().map(DSSFlow::getName).collect(Collectors.joining(", ")); + logger.info("user {} try to convert workflow(s) {} to SchedulerAppConn {}.", requestConversionWorkflow.getUserName(), + convertFlowStr, schedulerAppConnName); + SchedulerAppConn appConn = (SchedulerAppConn) AppConnManager.getAppConnManager().getAppConn(schedulerAppConnName); + AppInstance schedulerInstance = appConn.getAppDesc().getAppInstances().get(0); + DSSToRelConversionOperation operation = appConn.getOrCreateConversionStandard() + .getDSSToRelConversionService(schedulerInstance).getDSSToRelConversionOperation(); + DSSToRelConversionRequestRef requestRef = RequestRefUtils.getRequestRef(operation); + requestRef.setDSSProject((DSSProject) requestConversionWorkflow.getProject()) + .setUserName(requestConversionWorkflow.getUserName()) + .setWorkspace((Workspace) requestConversionWorkflow.getWorkspace()); + if(requestRef instanceof ProjectToRelConversionRequestRef) { + Map orchestrationMap = flowInfos.stream().filter(pair -> flows.contains(pair.getKey())) + .map(pair -> new ImmutablePair<>(pair.getKey().getName(), pair.getValue())) + .collect(HashMap::new, (map, pair) -> map.put(pair.getKey(), pair.getValue()), HashMap::putAll); + ((ProjectToRelConversionRequestRef) requestRef).setDSSOrcList(flows).setRefOrchestrationId(orchestrationMap); + } else if(requestRef instanceof OrchestrationToRelConversionRequestRef) { + flowInfos.stream().filter(pair -> pair.getKey() == flows.get(0)).forEach( pair -> + ((OrchestrationToRelConversionRequestRef) requestRef).setDSSOrchestration(pair.getKey()) + .setRefOrchestrationId(pair.getValue()) + ); + } + try{ + ResponseRef responseRef = operation.convert(requestRef); + if(responseRef.isFailed()) { + logger.error("user {} convert workflow(s) {} to {} failed, Reason: {}.", requestConversionWorkflow.getUserName(), + convertFlowStr, schedulerAppConnName, responseRef.getErrorMsg()); + return ResponseOperateOrchestrator.failed("workflow(s) " + convertFlowStr + " publish to " + schedulerAppConnName + "failed! Reason: " + + responseRef.getErrorMsg()); + } + return ResponseOperateOrchestrator.success(); + } catch (Exception e) { + logger.error("user {} convert workflow(s) {} to {} failed.", requestConversionWorkflow.getUserName(), + convertFlowStr, schedulerAppConnName, e); + return ResponseOperateOrchestrator.failed("Workflow(s) " + convertFlowStr + " publish to " + schedulerAppConnName + + "failed! Reason: " + ExceptionUtils.getRootCauseMessage(e)); + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java new file mode 100644 index 000000000..a04c24da6 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java @@ -0,0 +1,105 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestConvertOrchestrations; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface WorkFlowManager { + + /** + * create a new workflow + * @param userName + * @param workflowName + * @param contextIDStr + * @param description + * @param parentFlowID + * @param uses + * @return + */ + DSSFlow createWorkflow(String userName, + Long projectId, + String workflowName, + String contextIDStr, + String description, + Long parentFlowID, + String uses, + List linkedAppConnNames, + List dssLabels, + String orcVersion, + String schedulerAppConn) throws DSSErrorException, JsonProcessingException; + + + /** + * 根据已有的flow内容,拷贝一份新的flow,区分BML文件,用于绑定不同的版本 + * @param userName + * @param rootFlowId + * @param contextIdStr + * @param description + * @return + * @throws DSSErrorException + */ + DSSFlow copyRootFlowWithSubFlows(String userName, + Long rootFlowId, + Workspace workspace, + String projectName, + String contextIdStr, + String orcVersion, + String description, + List dssLabels) throws DSSErrorException, IOException; + + DSSFlow queryWorkflow(String userName, Long rootFlowId) throws DSSErrorException; + + + void updateWorkflow(String userName, + Long flowID, + String flowName, + String description, + String uses) throws DSSErrorException; + + + void deleteWorkflow(String userName, + Long flowID) throws DSSErrorException; + + + Map exportWorkflow(String userName, + Long flowID, + Long dssProjectId, + String projectName, + Workspace workspace, + List dssLabels) throws Exception; + + List importWorkflow(String userName, + String resourceId, + String bmlVersion, + DSSFlowImportParam dssFlowImportParam, + List dssLabels) throws Exception; + + ResponseOperateOrchestrator convertWorkflow(RequestConvertOrchestrations requestConversionWorkflow) throws DSSErrorException; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/conf/WorkflowServerSpringConfiguration.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/conf/WorkflowServerSpringConfiguration.java new file mode 100644 index 000000000..dc02f1609 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/conf/WorkflowServerSpringConfiguration.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.conf; + +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import com.webank.wedatasphere.dss.workflow.service.PublishService; +import com.webank.wedatasphere.dss.workflow.service.impl.PublishServiceImpl; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class WorkflowServerSpringConfiguration { + + @Bean + @ConditionalOnMissingBean + public PublishService createPublishService(@Autowired DSSFlowService dssFlowService, + @Autowired WorkFlowParser workFlowParser) { + PublishServiceImpl publishService = new PublishServiceImpl(); + publishService.setDssFlowService(dssFlowService); + publishService.setWorkFlowParser(workFlowParser); + return publishService; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java new file mode 100644 index 000000000..6209052c9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.constant; + +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; +import com.webank.wedatasphere.dss.common.conf.DSSCommonConf; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.CommonVars$; + + +public class DSSWorkFlowConstant { + public static final CommonVars DSS_EXPORT_ENV = CommonVars.apply("wds.dss.server.export.env", "DEV"); + public static final String PUBLISH_FLOW_REPORT_FORMATE = "工作流名:%s,版本号:%s,工作流内容为空,请自行修改或者删除"; + public static final Interner saveFlowLock = Interners.newWeakInterner(); + public static final CommonVars CACHE_TIMEOUT = CommonVars$.MODULE$.apply("wds.dss.server.cache.timeout", 1000 * 60 * 60); + public static final CommonVars PUBLISH_TIMEOUT = CommonVars$.MODULE$.apply("wds.dss.server.publish.timeout", 60 * 10); + public static final CommonVars NODE_EXPORT_IMPORT_THREAD_NUM = CommonVars.apply("wds.dss.server.workflow.exportImport.thread.num", 50); + public static final CommonVars NODE_EXPORT_IMPORT_TIMEOUT_MINUTES= CommonVars.apply("wds.dss.server.workflow.exportImport.timeout.minutes", 30); + /** + * 工作流编辑锁超时时间 + */ + public static final CommonVars DSS_FLOW_EDIT_LOCK_TIMEOUT = CommonVars$.MODULE$.apply("wds.dss.server.flow.edit.lock.timeout", 1000 * 60 * 10L); + + public static final String SPLIT = "_"; + + public static final String BDP_USER_TICKET_ID = DSSCommonConf.DSS_TOKEN_TICKET_KEY.getValue(); + + public static final String SCHEDULER_APP_CONN_NAME = "schedulerAppConnName"; + public static final String REF_PROJECT_ID_KEY = "refProjectId"; + public static final String TITLE_KEY = "title"; + + /** + * 用户已锁定编辑错误码 + */ + public static final int EDIT_LOCK_ERROR_CODE = 60056; + + /** + * 发布中的错误码 + */ + public static final String PUBLISHING_ERROR_CODE = "-999"; + + public static final CommonVars GOTO_SCHEDULER_CENTER_URL = CommonVars.apply("wds.dss.workflow.schedulerCenter.url", "/scheduleCenter"); + /** + * 仅仅用于兼容老的、已经创建的工作量,用于自动路由到一个默认的调度系统。 + */ + public static final CommonVars DEFAULT_SCHEDULER_APP_CONN = CommonVars.apply("wds.dss.workflow.scheduler.default", "schedulis"); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/DSSCSHelper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/DSSCSHelper.java new file mode 100644 index 000000000..ee4a3a9a1 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/DSSCSHelper.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.cs; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.apache.linkis.cs.client.service.CSTableService; +import org.apache.linkis.cs.common.entity.metadata.CSTable; +import org.apache.linkis.cs.common.entity.source.ContextKeyValue; +import org.apache.linkis.cs.common.exception.CSErrorException; +import org.apache.linkis.cs.common.utils.CSCommonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + + +public class DSSCSHelper { + + private final static Logger logger = LoggerFactory.getLogger(DSSCSHelper.class); + + private final static Gson gson = new Gson(); + + public static List getTableContextKeyValueList(String contextIDStr, String nodeName) { + try { + return CSTableService.getInstance().searchUpstreamTableKeyValue(contextIDStr, nodeName); + } catch (CSErrorException e) { + logger.error("Failed to get getTableContextKeyValueList: " + nodeName, e); + } + + logger.info("upstream tmp table is null"); + return null; + } + + + public static CSTable getCSTable(String contextIDStr, String contextKeyStr) { + + + try { + return CSTableService.getInstance().getCSTable(contextIDStr, contextKeyStr); + } catch (CSErrorException e) { + logger.error("Failed to get CSTable contextkey: " + contextKeyStr, e); + } + logger.info("This contextKey{} has no csTable ", contextKeyStr); + return null; + } + + public static String getContextIDStrByJson(String flowJson) { + + JsonObject jsonObject = gson.fromJson(flowJson, JsonObject.class); + if (!jsonObject.has(CSCommonUtils.CONTEXT_ID_STR) || null == jsonObject.get(CSCommonUtils.CONTEXT_ID_STR)){ + return null; + } + return jsonObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString(); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/service/CSTableService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/service/CSTableService.java new file mode 100644 index 000000000..60a0b5a91 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/service/CSTableService.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.cs.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + +import java.util.List; +import java.util.Map; + + +public interface CSTableService { + + List> queryTables(String dbName, String contextIDStr, String nodeName) throws DSSErrorException; + + List> queryTableMeta(String dbName, String contextIDStr, String contextKeyStr) throws DSSErrorException; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/service/CSTableServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/service/CSTableServiceImpl.java new file mode 100644 index 000000000..c9ad58119 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/cs/service/CSTableServiceImpl.java @@ -0,0 +1,88 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.cs.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.workflow.cs.DSSCSHelper; +import org.apache.linkis.cs.client.utils.SerializeHelper; +import org.apache.linkis.cs.common.entity.metadata.CSColumn; +import org.apache.linkis.cs.common.entity.metadata.CSTable; +import org.apache.linkis.cs.common.entity.source.ContextKeyValue; +import org.apache.linkis.cs.common.entity.source.ContextValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +@Service +public class CSTableServiceImpl implements CSTableService { + + private final static Logger logger = LoggerFactory.getLogger(CSTableServiceImpl.class); + + + @Override + public List> queryTables(String dbName, String contextIDStr, String nodeName) throws DSSErrorException { + List contextKeyValueList = DSSCSHelper.getTableContextKeyValueList(contextIDStr, nodeName); + List> tables = new ArrayList<>(); + if (null == contextKeyValueList || contextKeyValueList.isEmpty()) { + return tables; + } + for(ContextKeyValue contextKeyValue : contextKeyValueList){ + Map tableNode = new HashMap<>(); + ContextValue contextValue = contextKeyValue.getContextValue(); + if (null != contextKeyValue && null != contextValue.getValue()) { + try { + CSTable table = (CSTable) (contextValue.getValue()) ; + tableNode.put("tableName", table.getName()); + tableNode.put("isView", table.isView()); + tableNode.put("databaseName", dbName); + tableNode.put("createdBy", table.getCreator()); + tableNode.put("createdAt", table.getCreateTime()); + //contextKey 需要序列化 + tableNode.put("contextKey", SerializeHelper.serializeContextKey(contextKeyValue.getContextKey())); + tables.add(tableNode); + } catch (Exception e) { + logger.error("Failed to get table ", e); + } + } + + } + return tables; + } + + @Override + public List> queryTableMeta(String dbName, String contextIDStr, String contextKeyStr) throws DSSErrorException { + CSTable csTable = DSSCSHelper.getCSTable(contextIDStr, contextKeyStr); + CSColumn[] columns = csTable.getColumns(); + List> responseTables = new ArrayList<>(); + for (CSColumn column : columns) { + Map columnNode = new HashMap<>(); + columnNode.put("columnName", column.getName()); + columnNode.put("columnType", column.getType()); + columnNode.put("columnComment", column.getComment()); + columnNode.put("partitioned", false); + responseTables.add(columnNode); + } + return responseTables; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/FlowMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/FlowMapper.java new file mode 100644 index 000000000..3a8d9342b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/FlowMapper.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.dao; + + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.springframework.dao.DuplicateKeyException; + +import java.util.List; + + +public interface FlowMapper { + DSSFlow selectFlowByID(Long id); + + void insertFlow(DSSFlow dssFlow) throws DuplicateKeyException; + + void insertFlowRelation(@Param("flowID") Long flowID, @Param("parentFlowID") Long parentFlowID); + + void updateFlowBaseInfo(DSSFlow dssFlow) throws DuplicateKeyException; + + List selectSubFlowIDByParentFlowID(Long parentFlowID); + + List selectSavedSubFlowIDByParentFlowID(Long parentFlowID); + + void deleteFlowBaseInfo(Long flowID); + + void deleteFlowRelation(Long flowID); + + Long selectParentFlowIDByFlowID(Long flowID); + + Long getParentFlowID(Long flowID); + + List listFlowRelation(@Param("flowIDs") List flowIDs); + + void updateFlowInputInfo(DSSFlow dssFlow); + + DSSFlowRelation selectFlowRelation(@Param("flowID") Long flowID, @Param("parentFlowID") Long parentFlowID); + + @Select("select creator from dss_flow where id = #{flowId}") + String getCreatorById(@Param("flowId")Long flowId); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java new file mode 100644 index 000000000..c2f8ab5e3 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.workflow.dao; + +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; +import org.apache.ibatis.annotations.Param; +import org.springframework.dao.DuplicateKeyException; + + +public interface LockMapper { + + Integer compareAndSwap(DSSFlowEditLock lock); + + /** + * 获取数据库中isExpire字段为false的lock,只有一个,owner必须 + * + * @param flowID 工作流id + * @param owner 锁的持有者 + * @return DSSFlowEditLock + */ + DSSFlowEditLock getPersonalFlowEditLock(@Param("flowID") Long flowID, @Param("owner") String owner); + + Boolean flowNotExistEditLock(@Param("flowID") Long flowID, + @Param("flowVersion") String flowVersion, + @Param("projectVersionID") Long projectVersionID, + @Param("owner") String owner, + @Param("timeout") long timeout); + + DSSFlowEditLock getFlowEditLockByID(@Param("flowId") Long flowId); + + DSSFlowEditLock getFlowEditLockByLockContent(String lockContent); + + void insertLock(DSSFlowEditLock newLock) throws DuplicateKeyException; + + void clearExpire(@Param("expireTime") String expireTime, @Param("flowId") long flowId); + + void deleteALL(); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeInfoMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeInfoMapper.java new file mode 100644 index 000000000..3a950b7a4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeInfoMapper.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.dao; + + +import com.webank.wedatasphere.dss.workflow.entity.NodeGroup; +import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; +import java.util.List; + + +public interface NodeInfoMapper { + + List listNodeGroups(); + + NodeInfo getWorkflowNodeByType(String nodeType); + +} + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/flowMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/flowMapper.xml new file mode 100644 index 000000000..4870fbcbf --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/flowMapper.xml @@ -0,0 +1,141 @@ + + + + + + + + + + id,`name`,`state`,`source`,`description`,`create_time`,`creator`,`is_root_flow`,`rank`,`project_id`,`has_saved`,`uses`,`bml_version`,`resource_id`,`linked_appconn_names`,`dss_labels` + + + + + + + INSERT INTO dss_workflow () + VALUES + (#{id},#{name},#{state},#{source},#{description},#{createTime},#{creator},#{isRootFlow},#{rank},#{projectID},#{hasSaved},#{uses},#{bmlVersion},#{resourceId},#{linkedAppConnNames},#{dssLabels}) + + + + + + INSERT INTO dss_workflow_relation (flow_id,parent_flow_id) + VALUES + (#{flowID},#{parentFlowID}) + + + + UPDATE dss_workflow + + name=#{name}, + description=#{description}, + has_saved=#{hasSaved}, + uses=#{uses}, + linked_appconn_names=#{linkedAppConnNames}, + + WHERE id =#{id} + + + + + + + + DELETE + FROM + dss_workflow + WHERE id = #{flowID} + + + + DELETE + FROM + dss_workflow_relation + WHERE flow_id = #{flowID} + + + + + + + + + + + + + + + + UPDATE dss_workflow + + name=#{name}, + state = #{state}, + source = #{source}, + description=#{description}, + create_time=#{createTime}, + creator=#{creator}, + project_id=#{projectID}, + has_saved=#{hasSaved}, + uses=#{uses}, + resource_id=#{resourceId}, + bml_version=#{bmlVersion}, + linked_appconn_names=#{linkedAppConnNames}, + + WHERE id =#{id} + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml new file mode 100644 index 000000000..41e45c40c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml @@ -0,0 +1,76 @@ + + + + + + + + + + `id`,`flow_id`,`create_time`,`update_time`,`owner`,`is_expire`,`lock_content`,username + + + + update dss_workflow_edit_lock + + update_time = #{updateTime}, + is_expire = #{isExpire}, + + lock_content = #{lockContent} + + + + + + + + + + + + insert into dss_workflow_edit_lock() + values(#{id},#{flowID},#{createTime},#{updateTime},#{owner},#{isExpire},#{lockContent},#{username}) + + + + delete from dss_workflow_edit_lock where update_time <= #{expireTime} or flow_id = #{flowId} + + + + delete from dss_workflow_edit_lock + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/nodeInfoMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/nodeInfoMapper.xml new file mode 100644 index 000000000..805f903c2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/nodeInfoMapper.xml @@ -0,0 +1,142 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AbstractAppConnNode.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AbstractAppConnNode.java new file mode 100644 index 000000000..985b409a6 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AbstractAppConnNode.java @@ -0,0 +1,140 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + + +import java.util.Map; + + +public abstract class AbstractAppConnNode { + + private String projectName; + + private Long projectId; + + private String flowName; + + private Long flowId; + + private String nodeName; + + private String nodeId; + + private String nodeType; + + private String contextId; + + private Map jobContent; + + public AbstractAppConnNode(String projectName, long projectId, String flowName, long flowId, + String nodeName, String nodeTye, Map jobContent){ + this.projectId = projectId; + this.projectName = projectName; + this.flowName = flowName; + this.flowId = flowId; + this.nodeName = nodeName; + this.nodeType = nodeTye; + this.jobContent = jobContent; + } + + public AbstractAppConnNode(){ + + } + + + public String getContextId() { + return contextId; + } + + public void setContextId(String contextId) { + this.contextId = contextId; } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setFlowName(String flowName) { + this.flowName = flowName; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } + + public String getId() { + return this.nodeId; + } + + + public void setId(String id) { + this.nodeId = id; + } + + + public String getNodeType() { + return this.nodeType; + } + + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + + public String getName() { + return this.nodeName; + } + + + public void setName(String name) { + this.nodeName = name; + } + + + public Long getProjectId() { + return this.projectId; + } + + public String getProjectName() { + return this.projectName; + } + + + public String getFlowName() { + return this.flowName; + } + + + public Long getFlowId() { + return this.flowId; + } + + + public Map getJobContent() { + return this.jobContent; + } + + + public void setJobContent(Map jobContent) { + this.jobContent = jobContent; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMap.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMap.java new file mode 100644 index 000000000..32aa40703 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMap.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.List; + +public class AppMap { + public List getEdges() { + return edges; + } + + public void setEdges(List edges) { + this.edges = edges; + } + + public List getNodes() { + return nodes; + } + + public void setNodes(List nodes) { + this.nodes = nodes; + } + + List edges; + List nodes; +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapEdge.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapEdge.java new file mode 100644 index 000000000..f44cd5fd2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapEdge.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +public class AppMapEdge { + + private String source; + private String target; + private String sourceLocation; + private String targetLocation; + private String edgeType; + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getTarget() { + return target; + } + + public void setTarget(String target) { + this.target = target; + } + + public String getSourceLocation() { + return sourceLocation; + } + + public void setSourceLocation(String sourceLocation) { + this.sourceLocation = sourceLocation; + } + + public String getTargetLocation() { + return targetLocation; + } + + public void setTargetLocation(String targetLocation) { + this.targetLocation = targetLocation; + } + + public String getEdgeType() { + return edgeType; + } + + public void setEdgeType(String edgeType) { + this.edgeType = edgeType; + } +} + + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapFlowNode.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapFlowNode.java new file mode 100644 index 000000000..f547014ee --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapFlowNode.java @@ -0,0 +1,104 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.HashMap; + +public class AppMapFlowNode { + private String id; + private String desc; + private String businessTag; + private String appTag; + private String key; + private String title; + + private String isRootFlow; + + public String getIsRootFlow() { + return isRootFlow; + } + + public void setIsRootFlow(String isRootFlow) { + this.isRootFlow = isRootFlow; + } + + + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getBusinessTag() { + return businessTag; + } + + public void setBusinessTag(String businessTag) { + this.businessTag = businessTag; + } + + public String getAppTag() { + return appTag; + } + + public void setAppTag(String appTag) { + this.appTag = appTag; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public AppMapProjectInfo getProjectData() { + return projectData; + } + + public void setProjectData(AppMapProjectInfo projectData) { + this.projectData = projectData; + } + + private AppMapProjectInfo projectData; + private HashMap layout=new HashMap(){{ + put("height",120); + put("width",200); + put("x",""); + put("y","");} + }; +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapProjectInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapProjectInfo.java new file mode 100644 index 000000000..646a8ac89 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/AppMapProjectInfo.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +public class AppMapProjectInfo { + private long id; + private String desc; + private String title; + private boolean currentProject = false; + private String businessTag; + private String appTag; + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public boolean isCurrentProject() { + return currentProject; + } + + public void setCurrentProject(boolean currentProject) { + this.currentProject = currentProject; + } + + public String getBusinessTag() { + return businessTag; + } + + public void setBusinessTag(String businessTag) { + this.businessTag = businessTag; + } + + public String getAppTag() { + return appTag; + } + + public void setAppTag(String appTag) { + this.appTag = appTag; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/CommonAppConnNode.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/CommonAppConnNode.java new file mode 100644 index 000000000..a77c6109e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/CommonAppConnNode.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; +import java.util.Map; + + +public class CommonAppConnNode extends AbstractAppConnNode { + + private Map params; + private List dssLabels; + private Workspace workspace; + + public CommonAppConnNode(String projectName, long projectId , String flowName, long flowId, String nodeName, + String type, Map jobContent){ + super(projectName, projectId, flowName, flowId, nodeName, type, jobContent); + } + + public CommonAppConnNode(){ + + } + + public void setParams(Map params){ + this.params = params; + } + + public Map getParams(){ + return this.params; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/ContentLanguage.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/ContentLanguage.java new file mode 100644 index 000000000..481dcb735 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/ContentLanguage.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +public enum ContentLanguage { + en("en"), zh_CN("zh-CN"); + private String name; + + ContentLanguage(String name) { + this.name = name; + } + + public String getName() { + return this.name; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowEditLock.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowEditLock.java new file mode 100644 index 000000000..59b63597c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowEditLock.java @@ -0,0 +1,133 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.Date; + +/** + * 工作流编辑锁对象 + */ +public class DSSFlowEditLock { + + private Integer id; + + private Long flowID; + + private String flowVersion; + + private Date createTime; + + private Date updateTime; + + private String owner; + + private Integer lockStamp; + + private Boolean isExpire; + + private String lockContent;//uuid + + private String username; + + public DSSFlowEditLock() { + } + + public DSSFlowEditLock(String flowVersion, String lockContent) { + this.flowVersion = flowVersion; + this.lockContent = lockContent; + } + + public String getLockContent() { + return lockContent; + } + + public void setLockContent(String lockContent) { + this.lockContent = lockContent; + } + + public Boolean getExpire() { + return isExpire; + } + + public void setExpire(Boolean expire) { + isExpire = expire; + } + + public String getFlowVersion() { + return flowVersion; + } + + public void setFlowVersion(String flowVersion) { + this.flowVersion = flowVersion; + } + + public Integer getLockStamp() { + return lockStamp; + } + + public void setLockStamp(Integer lockStamp) { + this.lockStamp = lockStamp; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Long getFlowID() { + return flowID; + } + + public void setFlowID(Long flowID) { + this.flowID = flowID; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + @Override + public String toString() { + return "DSSFlowEditLock{" + + "id=" + id + + ", flowID=" + flowID + + ", flowVersion='" + flowVersion + '\'' + + ", createTime=" + createTime + + ", updateTime=" + updateTime + + ", owner='" + owner + '\'' + + ", lockStamp=" + lockStamp + + ", isExpire=" + isExpire + + ", lockContent='" + lockContent + '\'' + + ", username='" + username + '\'' + + '}'; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowImportParam.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowImportParam.java new file mode 100644 index 000000000..5835f4621 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowImportParam.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + + +public class DSSFlowImportParam { + + private Long projectID; + private String projectName; + private String orcVersion; + /** + * 目前只支持一个rootflow导入,只带了一个ContextID + */ + private String contextId; + private Workspace workspace; + private String userName; + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Long getProjectID() { + return projectID; + } + + public void setProjectID(Long projectID) { + this.projectID = projectID; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getOrcVersion() { + return orcVersion; + } + + public void setOrcVersion(String orcVersion) { + this.orcVersion = orcVersion; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public String getContextId() { + return contextId; + } + + public void setContextId(String contextId) { + this.contextId = contextId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/InputRelation.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/InputRelation.java new file mode 100644 index 000000000..ff82ae885 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/InputRelation.java @@ -0,0 +1,78 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + + +import com.webank.wedatasphere.dss.common.entity.IOEnv; + +public class InputRelation { + + private Long id; + private String type; + private IOEnv sourceEnv; + private Long sourceID; + private IOEnv targetEnv; + private Long targetID; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public IOEnv getSourceEnv() { + return sourceEnv; + } + + public void setSourceEnv(IOEnv sourceEnv) { + this.sourceEnv = sourceEnv; + } + + public Long getSourceID() { + return sourceID; + } + + public void setSourceID(Long sourceID) { + this.sourceID = sourceID; + } + + public IOEnv getTargetEnv() { + return targetEnv; + } + + public void setTargetEnv(IOEnv targetEnv) { + this.targetEnv = targetEnv; + } + + public Long getTargetID() { + return targetID; + } + + public void setTargetID(Long targetID) { + this.targetID = targetID; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeGroup.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeGroup.java new file mode 100644 index 000000000..f5bdd3adc --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeGroup.java @@ -0,0 +1,77 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +import java.io.Serializable; +import java.util.List; + +public class NodeGroup implements Serializable { + private Long id; + private String name; + private String nameEn; + private String description; + private Integer order; + private List nodes; + + public List getNodes() { + return nodes; + } + + public void setNodes(List nodes) { + this.nodes = nodes; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNameEn() { + return nameEn; + } + + public void setNameEn(String nameEn) { + this.nameEn = nameEn; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeInfo.java new file mode 100644 index 000000000..bba3bff4b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeInfo.java @@ -0,0 +1,165 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +import java.io.Serializable; +import java.util.List; + + +public class NodeInfo implements Serializable { + private Integer id; + /** + * 工作流节点图标路径,每个工作流节点的图标都存储在对应的 AppConn 之中 + */ + private String iconPath; + /** + * 工作流节点类型,必须以 linkis.[appconn/engineconn]开头。 + * 例如: + * 1. 如果是 sparkSQL,则必须是:linkis.spark.sql + * 2. 如果是 Visualis 的 widget,则必须是:linkis.appconn.visualis.widget + */ + private String nodeType; + /** + * 所关联的 AppConn 的 id + */ + private Integer appConnId; + /** + * 所关联的 AppConn 的 name + */ + private String appConnName; + /** + * 【保留字段】表示是否可以发布给调度系统 + */ + private Boolean submitToScheduler; + /** + * 节点是否支持拷贝 + */ + private Boolean enableCopy; + /** + * 该节点在拖拽到工作流画布前,是否需要先弹窗创建 + */ + private Boolean shouldCreationBeforeNode; + /** + * 是否支持跳转URL,即工作流节点是否支持Iframe嵌入 + */ + private Boolean supportJump; + /** + * 跳转类型:1 表示是外部节点,2 表示是 Scriptis 节点。如果 supportJump 为 false,则该字段无意义。 + */ + private int jumpType; + /** + * 节点名 + */ + private String name; + private List nodeUis; + + public List getNodeUis() { + return nodeUis; + } + + public void setNodeUis(List nodeUis) { + this.nodeUis = nodeUis; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getIconPath() { + return iconPath; + } + + public void setIconPath(String iconPath) { + this.iconPath = iconPath; + } + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public Integer getAppConnId() { + return appConnId; + } + + public void setAppConnId(Integer appConnId) { + this.appConnId = appConnId; + } + + public String getAppConnName() { + return appConnName; + } + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + public Boolean getSubmitToScheduler() { + return submitToScheduler; + } + + public void setSubmitToScheduler(Boolean submitToScheduler) { + this.submitToScheduler = submitToScheduler; + } + + public Boolean getEnableCopy() { + return enableCopy; + } + + public void setEnableCopy(Boolean enableCopy) { + this.enableCopy = enableCopy; + } + + public Boolean getShouldCreationBeforeNode() { + return shouldCreationBeforeNode; + } + + public void setShouldCreationBeforeNode(Boolean shouldCreationBeforeNode) { + this.shouldCreationBeforeNode = shouldCreationBeforeNode; + } + + public Boolean getSupportJump() { + return supportJump; + } + + public void setSupportJump(Boolean supportJump) { + this.supportJump = supportJump; + } + + public int getJumpType() { + return jumpType; + } + + public void setJumpType(int jumpType) { + this.jumpType = jumpType; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUi.java new file mode 100644 index 000000000..d4f1f9234 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUi.java @@ -0,0 +1,187 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + + +import java.io.Serializable; +import java.util.List; + +public class NodeUi implements Serializable { + private Long id; + private String key; + private String description; + private String descriptionEn; + private String lableName; + private String lableNameEn; + private NodeUiType uiType; + private Boolean required; + private String value; + private String defaultValue; + private Boolean isHidden; + private String condition; + private Boolean isAdvanced; + private Integer order; + private Boolean isBaseInfo; + private Integer nodeMenuType;//1为右边栏 + private String position; + + private List nodeUiValidates; + + public List getNodeUiValidates() { + return nodeUiValidates; + } + + public void setNodeUiValidates(List nodeUiValidates) { + this.nodeUiValidates = nodeUiValidates; + } + + public String getPosition() { + return position; + } + + public void setPosition(String position) { + this.position = position; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDescriptionEn() { + return descriptionEn; + } + + public void setDescriptionEn(String descriptionEn) { + this.descriptionEn = descriptionEn; + } + + public String getLableName() { + return lableName; + } + + public void setLableName(String lableName) { + this.lableName = lableName; + } + + public String getLableNameEn() { + return lableNameEn; + } + + public void setLableNameEn(String lableNameEn) { + this.lableNameEn = lableNameEn; + } + + public NodeUiType getUiType() { + return uiType; + } + + public void setUiType(NodeUiType uiType) { + this.uiType = uiType; + } + + public Boolean getRequired() { + return required; + } + + public void setRequired(Boolean required) { + this.required = required; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public Boolean getHidden() { + return isHidden; + } + + public void setHidden(Boolean hidden) { + isHidden = hidden; + } + + public String getCondition() { + return condition; + } + + public void setCondition(String condition) { + this.condition = condition; + } + + public Boolean getAdvanced() { + return isAdvanced; + } + + public void setAdvanced(Boolean advanced) { + isAdvanced = advanced; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } + + public Boolean getBaseInfo() { + return isBaseInfo; + } + + public void setBaseInfo(Boolean baseInfo) { + isBaseInfo = baseInfo; + } + + public Integer getNodeMenuType() { + return nodeMenuType; + } + + public void setNodeMenuType(Integer nodeMenuType) { + this.nodeMenuType = nodeMenuType; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiType.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiType.java new file mode 100644 index 000000000..9d66ac282 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiType.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +public enum NodeUiType { + Input, Text, Upload, Tag, Select, MultiSelect, Disable, Binding, MultiBinding +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidate.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidate.java new file mode 100644 index 000000000..83e73ba69 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidate.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +import java.io.Serializable; + +public class NodeUiValidate implements Serializable { + private Integer id; + private Integer uiId; + private NodeUiValidateType validateType; + private String validateRange; + private String errorMsg; + private String errorMsgEn; + private NodeUiValidateTrigger trigger; + + public NodeUiValidateTrigger getTrigger() { + return trigger; + } + + public void setTrigger(NodeUiValidateTrigger trigger) { + this.trigger = trigger; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Integer getUiId() { + return uiId; + } + + public void setUiId(Integer uiId) { + this.uiId = uiId; + } + + public NodeUiValidateType getValidateType() { + return validateType; + } + + public void setValidateType(NodeUiValidateType validateType) { + this.validateType = validateType; + } + + public String getValidateRange() { + return validateRange; + } + + public void setValidateRange(String validateRange) { + this.validateRange = validateRange; + } + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + public String getErrorMsgEn() { + return errorMsgEn; + } + + public void setErrorMsgEn(String errorMsgEn) { + this.errorMsgEn = errorMsgEn; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidateTrigger.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidateTrigger.java new file mode 100644 index 000000000..27ddb4ee6 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidateTrigger.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +public enum NodeUiValidateTrigger { + /** + * blur 失去焦点事件 + *

+ * change 事件 + */ + blur, change +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidateType.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidateType.java new file mode 100644 index 000000000..f36a7a2f2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUiValidateType.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity; + +public enum NodeUiValidateType { + /** + * 不校验,None不能和下面的校验类型搭配 + */ + None, + /** + * 整数类型 + */ + NumInterval, + /** + * 在这几个中 + */ + OFT, + /** + * 浮点数 + */ + FloatInterval, + /** + * 包含 + */ + Contains, + /** + * 正则 + */ + Regex, + /** + * 自定义函数..validate_range是函数名,由前台提供 + */ + Function, + /** + * 不为空 + */ + Required +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/AddFlowRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/AddFlowRequest.java new file mode 100644 index 000000000..8d01d81bb --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/AddFlowRequest.java @@ -0,0 +1,88 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class AddFlowRequest { + + private String name; + private String workspaceName; + private String projectName; + private String version; + private String description ; + private Long parentFlowID; + private String uses; + private LabelRouteVO labels; + private String userName; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Long getParentFlowID() { + return parentFlowID; + } + + public void setParentFlowID(Long parentFlowID) { + this.parentFlowID = parentFlowID; + } + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/AppConnNodeUrlRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/AppConnNodeUrlRequest.java new file mode 100644 index 000000000..38584ccc0 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/AppConnNodeUrlRequest.java @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +import java.util.Map; + +public class AppConnNodeUrlRequest { + + private Long projectID; + private Long flowId; + private String nodeType; + private Map params; + private LabelRouteVO labels; + + public Long getProjectID() { + return projectID; + } + + public void setProjectID(Long projectID) { + this.projectID = projectID; + } + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchDeleteAppConnNodeRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchDeleteAppConnNodeRequest.java new file mode 100644 index 000000000..3a4b8bc87 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchDeleteAppConnNodeRequest.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +import java.util.List; +import java.util.Map; + +public class BatchDeleteAppConnNodeRequest { + + private List> nodes; + private LabelRouteVO labels; + + public List> getNodes() { + return nodes; + } + + public void setNodes(List> nodes) { + this.nodes = nodes; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/CreateExternalNodeRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/CreateExternalNodeRequest.java new file mode 100644 index 000000000..262e807f7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/CreateExternalNodeRequest.java @@ -0,0 +1,65 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +import java.util.List; +import java.util.Map; + +public class CreateExternalNodeRequest { + + + private Long projectID; + private String nodeType; + private Long flowID; + private Map params; + private String nodeID; + private LabelRouteVO labels; + + public Long getProjectID() { + return projectID; + } + + public void setProjectID(Long projectID) { + this.projectID = projectID; + } + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public Long getFlowID() { + return flowID; + } + + public void setFlowID(Long flowID) { + this.flowID = flowID; + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + public String getNodeID() { + return nodeID; + } + + public void setNodeID(String nodeID) { + this.nodeID = nodeID; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DeleteFlowRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DeleteFlowRequest.java new file mode 100644 index 000000000..7aa27db94 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DeleteFlowRequest.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class DeleteFlowRequest { + + private Long id; + private Boolean sure; + private LabelRouteVO labels; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Boolean getSure() { + return sure; + } + + public void setSure(Boolean sure) { + this.sure = sure; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/GetExtraToolBarsRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/GetExtraToolBarsRequest.java new file mode 100644 index 000000000..7348ae2f4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/GetExtraToolBarsRequest.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class GetExtraToolBarsRequest { + private Long projectId; + private Long workflowId; + private LabelRouteVO labels; + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Long getWorkflowId() { + return workflowId; + } + + public void setWorkflowId(Long workflowId) { + this.workflowId = workflowId; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/PublishWorkflowRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/PublishWorkflowRequest.java new file mode 100644 index 000000000..8141d8234 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/PublishWorkflowRequest.java @@ -0,0 +1,66 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class PublishWorkflowRequest { + + private Long workflowId; + private String comment; + private LabelRouteVO labels; + + private String dssLabel; + private Integer orchestratorId; + private Integer orchestratorVersionId; + + public Integer getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Integer orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Integer getOrchestratorVersionId() { + return orchestratorVersionId; + } + + public void setOrchestratorVersionId(Integer orchestratorVersionId) { + this.orchestratorVersionId = orchestratorVersionId; + } + + + + public String getDssLabel() { + return dssLabel; + } + + public void setDssLabel(String dssLabel) { + this.dssLabel = dssLabel; + } + + + + public Long getWorkflowId() { + return workflowId; + } + + public void setWorkflowId(Long workflowId) { + this.workflowId = workflowId; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/QueryTableMetaRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/QueryTableMetaRequest.java new file mode 100644 index 000000000..2a83056a5 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/QueryTableMetaRequest.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class QueryTableMetaRequest { + + private String contextID; + private String nodeName; + private String contextKey; + private LabelRouteVO labels; + + public String getNodeName() { + return nodeName; + } + + public void setNodeName(String nodeName) { + this.nodeName = nodeName; + } + + public String getContextID() { + return contextID; + } + + public void setContextID(String contextID) { + this.contextID = contextID; + } + + public String getContextKey() { + return contextKey; + } + + public void setContextKey(String contextKey) { + this.contextKey = contextKey; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/SaveFlowRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/SaveFlowRequest.java new file mode 100644 index 000000000..2d04cb90a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/SaveFlowRequest.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class SaveFlowRequest { + + private Long id; + private String json; + private String workspaceName; + private String projectName; + private String flowEditLock; + private Boolean isNotHaveLock; + private LabelRouteVO labels; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getJson() { + return json; + } + + public void setJson(String json) { + this.json = json; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getFlowEditLock() { + return flowEditLock; + } + + public void setFlowEditLock(String flowEditLock) { + this.flowEditLock = flowEditLock; + } + + public Boolean getNotHaveLock() { + return isNotHaveLock; + } + + public void setNotHaveLock(Boolean notHaveLock) { + isNotHaveLock = notHaveLock; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/TablesRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/TablesRequest.java new file mode 100644 index 000000000..133748594 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/TablesRequest.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class TablesRequest { + + private String contextID; + private String nodeName; + private LabelRouteVO labels; + + public String getContextID() { + return contextID; + } + + public void setContextID(String contextID) { + this.contextID = contextID; + } + + public String getNodeName() { + return nodeName; + } + + public void setNodeName(String nodeName) { + this.nodeName = nodeName; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/UpdateExternalNodeRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/UpdateExternalNodeRequest.java new file mode 100644 index 000000000..df7c95ea1 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/UpdateExternalNodeRequest.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +import java.util.Map; + +public class UpdateExternalNodeRequest { + private Long projectID; + private String nodeType; + private Long flowID; + private LabelRouteVO labels; + private Map params; + private String name; + private String description; + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + public Long getProjectID() { + return projectID; + } + + public void setProjectID(Long projectID) { + this.projectID = projectID; + } + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public Long getFlowID() { + return flowID; + } + + public void setFlowID(Long flowID) { + this.flowID = flowID; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/UpdateFlowBaseInfoRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/UpdateFlowBaseInfoRequest.java new file mode 100644 index 000000000..54749a7df --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/UpdateFlowBaseInfoRequest.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class UpdateFlowBaseInfoRequest { + + private Long id; + private String name; + private String description ; + private String uses; + private LabelRouteVO labels; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/ExtraToolBarsVO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/ExtraToolBarsVO.java new file mode 100644 index 000000000..e42dce5ea --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/ExtraToolBarsVO.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.dss.workflow.entity.vo; + +public class ExtraToolBarsVO { + private String name; + private String url; + private String icon; + + public ExtraToolBarsVO(String name, String url, String icon) { + this.name = name; + this.url = url; + this.icon = icon; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeGroupVO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeGroupVO.java new file mode 100644 index 000000000..a27f099e7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeGroupVO.java @@ -0,0 +1,77 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity.vo; + +import java.util.List; + +public class NodeGroupVO implements Comparable { + private Long id; + /** + * title是前台字段..后台自动根据请求头返回中英文字段 + * chilren是为了适配前台插件的字段 + */ + private String title; + private String description; + private Integer order; + + private List children; + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } + + @Override + public int compareTo(NodeGroupVO o) { + return this.order - o.order; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeInfoVO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeInfoVO.java new file mode 100644 index 000000000..0e69d5607 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeInfoVO.java @@ -0,0 +1,108 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity.vo; + +import java.util.List; + +public class NodeInfoVO { + + private Integer id; + //icon + private String image; + //nodeType + private String type; + private Boolean enableCopy; + private Boolean shouldCreationBeforeNode; + private Boolean supportJump; // 是否支持跳转URL,即工作流节点是否支持Iframe嵌入 + private int jumpType; // 跳转类型:1 表示是外部节点,2 表示是 Scriptis 节点 + //name + private String title; + + private List nodeUiVOS; + + public List getNodeUiVOS() { + return nodeUiVOS; + } + + public void setNodeUiVOS(List nodeUiVOS) { + this.nodeUiVOS = nodeUiVOS; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Boolean getEnableCopy() { + return enableCopy; + } + + public void setEnableCopy(Boolean enableCopy) { + this.enableCopy = enableCopy; + } + + public Boolean getShouldCreationBeforeNode() { + return shouldCreationBeforeNode; + } + + public void setShouldCreationBeforeNode(Boolean shouldCreationBeforeNode) { + this.shouldCreationBeforeNode = shouldCreationBeforeNode; + } + + public Boolean getSupportJump() { + return supportJump; + } + + public void setSupportJump(Boolean supportJump) { + this.supportJump = supportJump; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public int getJumpType() { + return jumpType; + } + + public void setJumpType(int jumpType) { + this.jumpType = jumpType; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeUiVO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeUiVO.java new file mode 100644 index 000000000..cce659fdd --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeUiVO.java @@ -0,0 +1,178 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity.vo; + + + +import com.webank.wedatasphere.dss.workflow.entity.NodeUiType; + +import java.util.List; + +public class NodeUiVO implements Comparable { + private Long id; + private String key; + private String desc; + private String lableName; + private NodeUiType uiType; + private Boolean required; + private String value; + private String defaultValue; + private Boolean isHidden; + private String condition; + private Boolean isAdvanced; + private Integer order; + private Boolean isBaseInfo; + private Integer nodeMenuType;//1为右边栏 + private String position;//存放位置 nodes /nodes.params.startup/nodes.params.runtime + private List nodeUiValidateVOS; + + public List getNodeUiValidateVOS() { + return nodeUiValidateVOS; + } + + public void setNodeUiValidateVOS(List nodeUiValidateVOS) { + this.nodeUiValidateVOS = nodeUiValidateVOS; + } + + public String getPosition() { + return position; + } + + public void setPosition(String position) { + this.position = position; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getLableName() { + return lableName; + } + + public void setLableName(String lableName) { + this.lableName = lableName; + } + + public NodeUiType getUiType() { + return uiType; + } + + public void setUiType(NodeUiType uiType) { + this.uiType = uiType; + } + + public Boolean getRequired() { + return required; + } + + public void setRequired(Boolean required) { + this.required = required; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public Boolean getHidden() { + return isHidden; + } + + public void setHidden(Boolean hidden) { + isHidden = hidden; + } + + public String getCondition() { + return condition; + } + + public void setCondition(String condition) { + this.condition = condition; + } + + public Boolean getAdvanced() { + return isAdvanced; + } + + public void setAdvanced(Boolean advanced) { + isAdvanced = advanced; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } + + public Boolean getBaseInfo() { + return isBaseInfo; + } + + public void setBaseInfo(Boolean baseInfo) { + isBaseInfo = baseInfo; + } + + public Integer getNodeMenuType() { + return nodeMenuType; + } + + public void setNodeMenuType(Integer nodeMenuType) { + this.nodeMenuType = nodeMenuType; + } + + @Override + public int compareTo(NodeUiVO o) { + if (!this.isBaseInfo.equals(o.isBaseInfo)) { + return Boolean.compare(o.isBaseInfo, this.isBaseInfo); + } + return this.order - o.order; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeUiValidateVO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeUiValidateVO.java new file mode 100644 index 000000000..0e36ded73 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/vo/NodeUiValidateVO.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.entity.vo; + + +import com.webank.wedatasphere.dss.workflow.entity.NodeUiValidateTrigger; +import com.webank.wedatasphere.dss.workflow.entity.NodeUiValidateType; + +public class NodeUiValidateVO implements Comparable { + + private NodeUiValidateType validateType; //前台的自定义校验函数名,对应validateType + private String validateRange; + private NodeUiValidateTrigger trigger; //blur change + private String message; + + public NodeUiValidateType getValidateType() { + return validateType; + } + + public void setValidateType(NodeUiValidateType validateType) { + this.validateType = validateType; + } + + public NodeUiValidateTrigger getTrigger() { + return trigger; + } + + public void setTrigger(NodeUiValidateTrigger trigger) { + this.trigger = trigger; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getValidateRange() { + return validateRange; + } + + public void setValidateRange(String validateRange) { + this.validateRange = validateRange; + } + + + @Override + public int compareTo(NodeUiValidateVO o) { + //Required校验类型要在数组最前,否则前台无法渲染 + return o.validateType.ordinal() - this.validateType.ordinal(); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/exception/DSSWorkflowErrorException.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/exception/DSSWorkflowErrorException.java new file mode 100644 index 000000000..d85345895 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/exception/DSSWorkflowErrorException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class DSSWorkflowErrorException extends ErrorException { + /** + * Description: DSS框架的错误码,从81000到82000 + */ + public DSSWorkflowErrorException(int errorCode, String errorDesc){ + super(errorCode, errorDesc); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java new file mode 100644 index 000000000..87216241d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.export; + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; + +import java.io.IOException; +import java.util.List; + +public interface MetaExportService { + + + void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaWriter.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaWriter.java new file mode 100644 index 000000000..41e23de54 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaWriter.java @@ -0,0 +1,164 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.export; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class MetaWriter { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final Class tClass; + private final String tableName; + private List datas = new ArrayList<>(); + private List ignoreFields = new ArrayList<>(); + private List fields = null; + private List comments = new ArrayList<>(); + private List table = new ArrayList<>(); + private String commentPrefix = "#"; + private String seperator = "|"; + + public MetaWriter(Class tClass, String tableName) { + this.tClass = tClass; + this.tableName = tableName; + } + + public static MetaWriter of(String tableName, Class tClass) { + return new MetaWriter<>(tClass, tableName); + } + + public MetaWriter ignore(String... fields) { + ignoreFields.addAll(Arrays.stream(fields).collect(Collectors.toList())); + return this; + } + + public MetaWriter data(List datas) { + this.datas.addAll(datas); + return this; + } + + public MetaWriter data(T data) { + this.datas.add(data); + return this; + } + + public MetaWriter comment(String... comments) { + this.comments.addAll(Arrays.stream(comments).collect(Collectors.toList())); + return this; + } + + /** + * 上传bml + * + * @return + */ + public InputStream write() { + writeComment(); + writeHead(); + writeBody(); + //table 添加换行符进行转流 + String tableStr = table.stream().reduce((a, b) -> a + "\n" + b).orElse("") + "\n"; + logger.info("\n" + tableStr); + return new ByteArrayInputStream(tableStr.getBytes()); + } + + /** + * 写入本地 + * + * @param outputStream + */ + public void write(OutputStream outputStream) throws IOException { + writeComment(); + writeHead(); + writeBody(); + IOUtils.writeLines(table, "\n", outputStream); + } + + private void writeBody() { + datas.forEach(DSSExceptionUtils.handling(this::writeBody)); + } + + private void writeBody(T t) throws NoSuchFieldException, IllegalAccessException { + ArrayList line = new ArrayList<>(); + for (String field : fields) { + Field declaredField = tClass.getDeclaredField(field); + declaredField.setAccessible(true); + Object o = declaredField.get(t); + if (o != null) { + line.add(o.toString()); + } else { + line.add(null); + } + } + // table.add(reduce(line)); + //防止出现描述内容带有换行符,导致发布时导出的内容格式错乱 + table.add(reduce(line).replaceAll("[\n\r]"," ")); + } + + private String reduce(List strs) { + return strs.stream().reduce((a, b) -> a + seperator + b).orElse(""); + } + + /** + * 写表的头部 + * 1.过滤ignore的属性 + * 2.驼峰转mysql的 _ (暂时略过) + * 3.写入List table + */ + private void writeHead() { + fields = Arrays.stream(tClass.getDeclaredFields()) + .map(Field::getName) + .filter(n -> !ignoreFields.contains(n)) + .collect(Collectors.toList()); + table.add(reduce(fields)); + } + + /** + * 写comment,包括表名,class名,和外部自定义的comment + */ + private void writeComment() { + table.add(connectCommentPrefix(String.format("tableName:%s", tableName))); + table.add(connectCommentPrefix(String.format("class:%s", tClass.getName()))); + table.add(connectCommentPrefix(String.format("env:%s", DSSWorkFlowConstant.DSS_EXPORT_ENV.getValue()))); + comments.stream().map(this::connectCommentPrefix).forEach(table::add); + } + + /** + * str前加上comment标识 + * + * @param str + * @return + */ + private String connectCommentPrefix(String str) { + return commentPrefix + str; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java new file mode 100644 index 000000000..8aa063b23 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.export; + +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; + +public interface NodeExportService { + void downloadNodeResourceToLocal(String userName, DSSNode dwsNode, String savePath); + void downloadAppConnResourceToLocal(String userName, Long projectId, String projectName, DSSNode dwsNode, String savePath, Workspace workspace, List dssLabels) throws Exception; +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java new file mode 100644 index 000000000..15b323302 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.export; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; + +public interface WorkFlowExportService { + + String exportFlowInfo(Long dssProjectId, String projectName, long rootFlowId, String userName, Workspace workspace, List dssLabels) throws Exception; + + void exportFlowResources(String userName, Long projectId, String projectName, String projectSavePath, String flowJson, String flowName, Workspace workspace,List dssLabels) throws Exception; + + String downloadFlowJsonFromBml(String userName, String resourceId, String version, String savePath); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java new file mode 100644 index 000000000..263c2d05d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java @@ -0,0 +1,78 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.export.impl; + +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import com.webank.wedatasphere.dss.workflow.io.export.MetaExportService; +import com.webank.wedatasphere.dss.workflow.io.export.MetaWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import java.io.*; +import java.util.List; + +@Service +public class MetaExportServiceImpl implements MetaExportService { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + + private final String fileName = "meta.txt"; + + + @Override + public void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException { + + try ( + OutputStream outputStream = generateOutputStream(savePath) + ) { + exportFlowBaseInfo(allDSSFlows, outputStream); + exportFlowRelation(allFlowRelations, outputStream); + } + } + + private OutputStream generateOutputStream(String basePath) throws IOException { + return IoUtils.generateExportOutputStream(basePath + File.separator + fileName); + } + + private void exportFlowBaseInfo(List DSSFlows, OutputStream outputStream) throws IOException { + + MetaWriter.of("dss_flow", DSSFlow.class).data(DSSFlows).write(outputStream); + + } + + private InputStream exportFlowBaseInfo(List DSSFlows) throws IOException { + + return MetaWriter.of("dss_flow", DSSFlow.class).data(DSSFlows).write(); + + } + + private void exportFlowRelation(List flowRelations, OutputStream outputStream) throws IOException { + + MetaWriter.of("dss_workflow_relation", DSSFlowRelation.class).data(flowRelations).write(outputStream); + + } + + private InputStream exportFlowRelation(List flowRelations) throws IOException { + + return MetaWriter.of("dss_workflow_relation", DSSFlowRelation.class).data(flowRelations).write(); + + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java new file mode 100644 index 000000000..a0ada025f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java @@ -0,0 +1,103 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.export.impl; + +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.development.ref.ExportResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.dao.NodeInfoMapper; +import com.webank.wedatasphere.dss.workflow.entity.CommonAppConnNode; +import com.webank.wedatasphere.dss.workflow.io.export.NodeExportService; +import com.webank.wedatasphere.dss.workflow.service.BMLService; +import com.webank.wedatasphere.dss.workflow.service.WorkflowNodeService; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.Closeable; +import java.io.File; +import java.io.InputStream; +import java.util.List; +import java.util.Map; + + +@Service +public class NodeExportServiceImpl implements NodeExportService { + + private final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private BMLService bmlService; + @Autowired + private NodeInfoMapper nodeInfoMapper; + @Autowired + private WorkflowNodeService workflowNodeService; + + @Override + public void downloadNodeResourceToLocal(String userName, DSSNode dwsNode, String savePath) { + List resources = dwsNode.getResources(); + if (resources != null) { + resources.forEach(x -> { + // TODO: 2020/6/9 防止前台传的 resources:{{}],后期要去掉 + if (x.getResourceId() != null && x.getFileName() != null && x.getVersion() != null) { + String nodeResourcePath = savePath + File.separator + x.getResourceId() + "_" + x.getVersion() + ".re"; + bmlService.downloadToLocalPath(userName, x.getResourceId(), x.getVersion(), nodeResourcePath); + } else { + LOGGER.warn("Illegal resource information"); + LOGGER.warn("username:{},nodeId:{},nodeName:{},fileName:{},version:{},resourceId:{}", userName, dwsNode.getId(), dwsNode.getName(), x.getFileName(), x.getVersion(), x.getResourceId()); + } + }); + } + } + + @Override + public void downloadAppConnResourceToLocal(String userName, Long projectId, String projectName, DSSNode dwsNode, + String savePath, Workspace workspace, List dssLabels) throws Exception { + CommonAppConnNode node = new CommonAppConnNode(); + node.setJobContent(dwsNode.getJobContent()); + node.setProjectId(projectId); + node.setWorkspace(workspace); + node.setDssLabels(dssLabels); + node.setNodeType(dwsNode.getNodeType()); + node.setName(dwsNode.getName()); + node.setId(dwsNode.getId()); + ExportResponseRef responseRef = workflowNodeService.exportNode(userName, node); + Map resourceMap = responseRef.getResourceMap(); + String nodeResourcePath = savePath + File.separator + dwsNode.getId() + ".appconnre"; + if(responseRef.isLinkisBMLResources()) { + String resourceId = resourceMap.get(ImportRequestRef.RESOURCE_ID_KEY).toString(); + String version = resourceMap.get(ImportRequestRef.RESOURCE_VERSION_KEY).toString(); + bmlService.downloadToLocalPath(userName, resourceId, version, nodeResourcePath); + } else { + InputStream inputStream = (InputStream) resourceMap.get(ImportRequestRef.INPUT_STREAM_KEY); + Closeable closeable = (Closeable) resourceMap.get(ImportRequestRef.CLOSEABLE_KEY); + try { + FileUtils.copyInputStreamToFile(inputStream, new File(nodeResourcePath)); + } finally { + IOUtils.closeQuietly(inputStream); + IOUtils.closeQuietly(closeable); + } + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java new file mode 100644 index 000000000..7e2eaa3b2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java @@ -0,0 +1,316 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.export.impl; + + +import com.webank.wedatasphere.dss.common.entity.IOType; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; +import com.webank.wedatasphere.dss.common.entity.node.DSSEdgeDefault; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.entity.node.Node; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.dao.NodeInfoMapper; +import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; +import com.webank.wedatasphere.dss.workflow.io.export.MetaExportService; +import com.webank.wedatasphere.dss.workflow.io.export.NodeExportService; +import com.webank.wedatasphere.dss.workflow.io.export.WorkFlowExportService; +import com.webank.wedatasphere.dss.workflow.service.BMLService; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.util.*; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.NODE_EXPORT_IMPORT_TIMEOUT_MINUTES; +import static com.webank.wedatasphere.dss.workflow.scheduler.DssJobThreadPool.nodeExportThreadPool; + + +@Service +public class WorkFlowExportServiceImpl implements WorkFlowExportService { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private BMLService bmlService; + + @Autowired + private WorkFlowParser workFlowParser; + + @Autowired + private NodeExportService nodeExportService; + + @Autowired + private MetaExportService metaExportService; + + @Autowired + private FlowMapper flowMapper; + + @Autowired + private NodeInfoMapper nodeInfoMapper; + + @Autowired + private DSSFlowService flowService; + + + @Override + public String exportFlowInfo(Long dssProjectId, String projectName, long rootFlowId, String userName, Workspace workspace, List dssLabels) throws Exception { + //获取rootFlow,和所有子Flow + DSSFlow rootFlow = flowMapper.selectFlowByID(rootFlowId); + List dssFlowList = new ArrayList<>(); + //生成rootflow及所有子flow + dssFlowList.add(rootFlow); + getAllDssFlowsByRootflowId(rootFlow, dssFlowList); + //生成rootflow及所有子flow的Relations + List flowIds = dssFlowList.stream().map(DSSFlow::getId).collect(Collectors.toList()); + List flowRelations = flowIds.isEmpty() ? new ArrayList<>() : flowMapper.listFlowRelation(flowIds); + String flowExportSaveBasePath = IoUtils.generateIOPath(userName, projectName, ""); + //标记当前导出为project导出 + IoUtils.generateIOType(IOType.FLOW, flowExportSaveBasePath); + //标记当前导出环境 + IoUtils.generateIOEnv(flowExportSaveBasePath); + metaExportService.exportFlowBaseInfo(dssFlowList, flowRelations, flowExportSaveBasePath); + logger.info(userName + "-开始导出Flow:" + rootFlow.getName()); + List dssFlows = new ArrayList<>(); + + for (DSSFlow dssFlow : dssFlowList) { + if (dssFlow.getRootFlow()) { + String savePath = flowExportSaveBasePath + File.separator + dssFlow.getName() + File.separator + dssFlow.getName() + ".json"; + //导出工作流json文件 + String flowJson = downloadFlowJsonFromBml(userName, dssFlow.getResourceId(), dssFlow.getBmlVersion(), savePath); + if (!dssFlow.getHasSaved()) { + logger.info("工作流{}从未保存过,忽略", dssFlow.getName()); + } else if (StringUtils.isNotBlank(flowJson)) { + exportFlowResources(userName, dssProjectId, projectName, flowExportSaveBasePath, flowJson, dssFlow.getName(), workspace, dssLabels); + exportAllSubFlows(userName, dssFlow, dssProjectId, projectName, flowExportSaveBasePath, workspace, dssLabels); + dssFlows.add(dssFlow); + } else { + String warnMsg = String.format(DSSWorkFlowConstant.PUBLISH_FLOW_REPORT_FORMATE, dssFlow.getName(), dssFlow.getBmlVersion()); + logger.info(warnMsg); + throw new DSSErrorException(90033, warnMsg); + } + } + } + if (dssFlows.isEmpty()) { + throw new DSSErrorException(90037, "该工程没有可以导出的工作流,请检查工作流是否都为空"); + } + //打包导出工程 + return ZipHelper.zipExportProject(flowExportSaveBasePath); + } + + + private void exportAllSubFlows(String userName, DSSFlow dssFlowParent, Long projectId, String projectName, + String projectExportBasePath, Workspace workspace, List dssLabels) throws Exception { + List subFlows = dssFlowParent.getChildren(); + if (subFlows != null) { + for (DSSFlow subFlow : subFlows) { + String savePath = projectExportBasePath + File.separator + subFlow.getName() + File.separator + subFlow.getName() + ".json"; + //导出子flow的json文件 + String flowJson = downloadFlowJsonFromBml(userName, subFlow.getResourceId(), subFlow.getBmlVersion(), savePath); + if (!subFlow.getHasSaved()) { + logger.info("工作流{}从未保存过,忽略", subFlow.getName()); + } else if (StringUtils.isNotBlank(flowJson)) { + exportFlowResources(userName, projectId, projectName, projectExportBasePath, flowJson, subFlow.getName(), workspace, dssLabels); + exportAllSubFlows(userName, subFlow, projectId, projectName, projectExportBasePath, workspace, dssLabels); + } else { + String warnMsg = String.format(DSSWorkFlowConstant.PUBLISH_FLOW_REPORT_FORMATE, subFlow.getName(), subFlow.getBmlVersion()); + logger.info(warnMsg); + throw new DSSErrorException(90014, warnMsg); + } + } + } + } + + + private String genWorkFlowExportDir(String projectExportPath, String flowName) { + return projectExportPath + File.separator + flowName; + } + + @Override + public void exportFlowResources(String userName, Long projectId, String projectName, + String projectSavePath, String flowJson, String flowName, + Workspace workspace, List dssLabels) throws Exception { + String workFlowExportPath = genWorkFlowExportDir(projectSavePath, flowName); + String workFlowResourceSavePath = workFlowExportPath + File.separator + "resource"; + String appConnResourceSavePath = workFlowExportPath + File.separator + "appconn-resource"; + if (StringUtils.isNotEmpty(workFlowExportPath)) { + //导出工作流资源文件 + List resources = workFlowParser.getWorkFlowResources(flowJson); + if (resources != null) { + resources.forEach(resource -> { + downloadFlowResourceFromBml(userName, resource, workFlowResourceSavePath); + }); + } + + //导出工作流节点资源文件,工作流节点appconn文件 + List nodes = workFlowParser.getWorkFlowNodes(flowJson); + if (nodes != null) { + for (DSSNode node : nodes) { + nodeExportService.downloadNodeResourceToLocal(userName, node, workFlowResourceSavePath); + NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(node.getNodeType()); + if (Boolean.TRUE.equals(nodeInfo.getSupportJump()) && nodeInfo.getJumpType() == 1) { + logger.info("node.getJobContent() is :{}", node.getJobContent()); + nodeExportService.downloadAppConnResourceToLocal(userName, projectId, projectName, node, appConnResourceSavePath, workspace, dssLabels); + } + } + } + + } else { + throw new DSSErrorException(90067, "工作流导出生成路径为空"); + } + } + + public void exportFlowResources_for_multi_thread(String userName, Long projectId, String projectName, + String projectSavePath, String flowJson, String flowName, + Workspace workspace, List dssLabels) throws Exception { + String workFlowExportPath = genWorkFlowExportDir(projectSavePath, flowName); + String workFlowResourceSavePath = workFlowExportPath + File.separator + "resource"; + String appConnResourceSavePath = workFlowExportPath + File.separator + "appconn-resource"; + if (StringUtils.isNotEmpty(workFlowExportPath)) { + //导出工作流资源文件 + List resources = workFlowParser.getWorkFlowResources(flowJson); + if (resources != null) { + resources.forEach(resource -> { + downloadFlowResourceFromBml(userName, resource, workFlowResourceSavePath); + }); + } + + //导出工作流节点资源文件,工作流节点appconn文件 + List nodes = workFlowParser.getWorkFlowNodes(flowJson); + if (CollectionUtils.isNotEmpty(nodes)) { + List edges = workFlowParser.getWorkFlowEdges(flowJson); + Map waitingNodes = nodes.stream().collect(Collectors.toConcurrentMap(Node::getId, t -> t)); + Set completedNodes = Collections.synchronizedSet(new HashSet<>()); + List failedNodes = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch countDownLatch = new CountDownLatch(nodes.size()); + List futureTaskList = Collections.synchronizedList(new ArrayList<>()); +// Map.Entry entry; + while (!waitingNodes.isEmpty()) { + //若有失败节点,直接失败,中断其他节点线程 + if (!failedNodes.isEmpty()) { + for (FutureTask futureTask : futureTaskList) { + futureTask.cancel(true); + } + throw new DSSErrorException(90070, "有节点导出失败,请重试: " + failedNodes); + } + for (Map.Entry entry : waitingNodes.entrySet()) { +// entry = stringDSSNodeEntry; + String nodeId = entry.getKey(); + DSSNode node = entry.getValue(); + //todo 缓存 + Set dependNodes = edges.stream().filter(l -> l.getTarget().equals(nodeId)).map(DSSEdge::getSource).collect(Collectors.toSet()); + boolean canSubmit = true; + if (!dependNodes.isEmpty()) { + for (String n : dependNodes) { + if (!completedNodes.contains(n)) { + canSubmit = false; + break; + } + } + } + if (canSubmit) { + FutureTask futureTask = new FutureTask<>(() -> { + boolean result = false; + try { + nodeExportService.downloadNodeResourceToLocal(userName, node, workFlowResourceSavePath); + NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(node.getNodeType()); + if (Boolean.TRUE.equals(nodeInfo.getSupportJump()) && nodeInfo.getJumpType() == 1) { + logger.info("node.getJobContent() is :{}", node.getJobContent()); + nodeExportService.downloadAppConnResourceToLocal(userName, projectId, projectName, node, appConnResourceSavePath, workspace, dssLabels); + } + completedNodes.add(nodeId); + result = true; + } catch (Exception e) { + failedNodes.add(node.getName()); + logger.error("failed to export node:{}", node.getName(), e); + } finally { + countDownLatch.countDown(); + } + return result; + }); + waitingNodes.remove(nodeId); + nodeExportThreadPool.submit(futureTask); + futureTaskList.add(futureTask); + } + } + Thread.sleep(10L); + } + boolean success = false; + try { + success = countDownLatch.await(NODE_EXPORT_IMPORT_TIMEOUT_MINUTES.getValue(), TimeUnit.MINUTES); + } catch (InterruptedException e) { + logger.error("failed to export node for workflow:{}", flowName, e); + throw new DSSErrorException(90071, "导出节点超时!"); + } + if (!failedNodes.isEmpty()) { + throw new DSSErrorException(90070, "有节点导出失败,请重试: " + failedNodes); + } + } + + } else { + throw new DSSErrorException(90067, "工作流导出生成路径为空"); + } + } + + @Override + public String downloadFlowJsonFromBml(String userName, String resourceId, String version, String savePath) { + return bmlService.downloadAndGetFlowJson(userName, resourceId, version, savePath); + } + + private String downloadFlowResourceFromBml(String userName, Resource resource, String savePath) { + String flowResourcePath = savePath + File.separator + resource.getResourceId() + ".re"; + return bmlService.downloadToLocalPath(userName, resource.getResourceId(), resource.getVersion(), flowResourcePath); + } + + private void getAllDssFlowsByRootflowId(DSSFlow parentFlow, List flowList) { + List subFlowIds = flowMapper.selectSavedSubFlowIDByParentFlowID(parentFlow.getId()); + if (subFlowIds.size() > 0) { + for (Long subFlowId : subFlowIds) { + + DSSFlow subDssFlow = flowMapper.selectFlowByID(subFlowId); + if (!subDssFlow.getHasSaved()) { + logger.info("工作流{}从未保存过,忽略", subDssFlow.getName()); + } else { + flowList.add(subDssFlow); + parentFlow.addChildren(subDssFlow); + getAllDssFlowsByRootflowId(subDssFlow, flowList); + } + } + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java new file mode 100644 index 000000000..506de99e4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.input; + + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; + +import java.io.IOException; +import java.util.List; + +public interface MetaInputService { + + List inputFlow(String basePath) throws IOException; + + List inputFlowRelation(String basePath) throws IOException; + + public List inputFlowRelation_for_0_x(String basePath) throws IOException; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaReader.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaReader.java new file mode 100644 index 000000000..3d14832bf --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaReader.java @@ -0,0 +1,165 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.input; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Field; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +public class MetaReader { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final Class tClass; + private final String tableName; + private String commentPrefix = "#"; + private String seperator = "\\|"; + private List datas = new ArrayList<>(); + private List fields = null; + private List comments = new ArrayList<>(); + private List> body = new ArrayList<>(); + private boolean finded = false; + private boolean firstLine = true; + + public MetaReader(Class tClass, String tableName) { + this.tClass = tClass; + this.tableName = tableName; + } + + public static MetaReader of(String tableName, Class tClass) { + return new MetaReader<>(tClass, tableName); + } + + public List read(InputStream inputStream) throws IOException { + try (InputStreamReader streamReader = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(streamReader);) { + readTable(reader); + } + readT(); + return datas; + } + + public String read(InputStream inputStream, String key) throws IOException { + try (InputStreamReader streamReader = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(streamReader);) { + readTable(reader); + } + String comment = comments.stream().filter(c -> c.contains(key + ":")).findFirst().orElse(""); + String[] split = comment.split(":"); + if (split.length > 1) return split[1]; + return ""; + } + + private void readT() { + body.stream().map(DSSExceptionUtils.map(this::lineToT)).forEach(datas::add); + } + + private T lineToT(List list) throws IllegalAccessException, InstantiationException, NoSuchFieldException, ParseException { + T t = tClass.newInstance(); + for (int i = 0; i < list.size(); i++) { + String valueStr = list.get(i); + if ("null".equalsIgnoreCase(valueStr)) continue; + Field declaredField = tClass.getDeclaredField(fields.get(i)); + declaredField.setAccessible(true); + Object value = null; + String type = declaredField.getType().getSimpleName(); + switch (type) { + case "String": + value = valueStr; + break; + case "Date": + value = new SimpleDateFormat("EEE MMM dd HH:mm:ss ZZZ yyyy", Locale.ENGLISH).parse(valueStr); + break; + case "Long": + value = Long.valueOf(valueStr); + break; + case "Boolean": + value = Boolean.valueOf(valueStr); + break; + case "Integer": + value = Integer.valueOf(valueStr); + break; + default: + logger.warn(String.format("unsupport type %s", type)); + } + declaredField.set(t, value); + } + return t; + } + + private void readTable(BufferedReader reader) throws IOException { + String line = null; + while ((line = reader.readLine()) != null) { + if (!finded && !shut(line)) { + continue; + } + if (finded && isTableName(line)) { + break; + } + if (shut(line)) { + finded = true; + continue; + } + // TODO: 2020/3/9 + if (isComment(line)) { + comments.add(line); + continue; + } + if (firstLine) { + //handle head + fields = Arrays.stream(line.split(seperator)).collect(Collectors.toList()); + firstLine = false; + continue; + } + body.add(Arrays.stream(line.split(seperator)).collect(Collectors.toList())); + //handle body + } + } + + private boolean isComment(String str) { + return str.startsWith(commentPrefix); + } + + private boolean isTableName(String str) { + return isComment(str) && str.contains(commentPrefix + "tableName:"); + } + + private boolean isClass(String str) { + return isComment(str) && str.contains(commentPrefix + "class:"); + } + + private String getComment(String str) { + return str.substring(1); + } + + private boolean shut(String str) { + return isTableName(str) && getComment(str).equals(String.format("tableName:%s", tableName)); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java new file mode 100644 index 000000000..e01c1f092 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.input; + + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; + +import java.io.IOException; +import java.util.List; + +public interface NodeInputService { + + String uploadResourceToBml(String userName, String nodeJson, String inputResourcePath, String projectName) throws IOException; + + String uploadAppConnResource(String userName, String projectName, DSSFlow dssFlow, + String nodeJson, String flowContextId, String appConnResourcePath, + Workspace workspace, String orcVersion, List dssLabels) throws DSSErrorException,IOException; + + String updateNodeSubflowID(String nodeJson, long subflowID) throws IOException; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java new file mode 100644 index 000000000..b954d8d28 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.input; + + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; + +import java.io.IOException; +import java.util.List; + +public interface WorkFlowInputService { + + /** + * + * @param userName + * @param dssFlow + * @param projectName + * @param inputProjectPath + * @param parentFlowId + * @param workspace + * @param orcVersion + * @throws DSSErrorException + + * @throws IOException + */ + void inputWorkFlow(String userName, DSSFlow dssFlow, String projectName, + String inputProjectPath, Long parentFlowId, Workspace workspace, + String orcVersion, String contextId, List dssLabels) throws DSSErrorException,IOException; + + /** + * save flow to db + * @param projectId + * @param userName + * @param dssFlows + * @param DSSFlowRelations + * @return + */ + List persistenceFlow(Long projectId, String userName, List dssFlows, List DSSFlowRelations); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java new file mode 100644 index 000000000..0d181ebdc --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.input.impl; + +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import com.webank.wedatasphere.dss.workflow.io.input.MetaInputService; +import com.webank.wedatasphere.dss.workflow.io.input.MetaReader; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + + +@Service +public class MetaInputServiceImpl implements MetaInputService { + + // TODO: 2020/3/13 防止表结构发生改变的version 字段的添加 + + private final String fileName = "meta.txt"; + + + @Override + public List inputFlow(String basePath) throws IOException { + try (InputStream inputStream = generateInputstream(basePath)) { + return MetaReader.of("dss_flow", DSSFlow.class).read(inputStream); + } + } + + @Override + public List inputFlowRelation(String basePath) throws IOException { + try (InputStream inputStream = generateInputstream(basePath)) { + return MetaReader.of("dss_workflow_relation", DSSFlowRelation.class).read(inputStream); + } + } + + @Override + public List inputFlowRelation_for_0_x(String basePath) throws IOException { + try (InputStream inputStream = generateInputstream(basePath)) { + return MetaReader.of("dss_flow_relation", DSSFlowRelation.class).read(inputStream); + } + } + + /** + * 获取inputStream + * + * @param basePath + * @return + * @throws FileNotFoundException + */ + private InputStream generateInputstream(String basePath) throws IOException { + return IoUtils.generateInputInputStream(basePath + File.separator + fileName); + } + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java new file mode 100644 index 000000000..906dbb713 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.input.impl; + + +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.parser.NodeParser; +import com.webank.wedatasphere.dss.workflow.entity.CommonAppConnNode; +import com.webank.wedatasphere.dss.workflow.io.input.NodeInputService; +import com.webank.wedatasphere.dss.workflow.service.BMLService; +import com.webank.wedatasphere.dss.workflow.service.WorkflowNodeService; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; + +@Service +public class NodeInputServiceImpl implements NodeInputService { + @Autowired + private BMLService bmlService; + + @Autowired + private NodeParser nodeParser; + + @Autowired + private WorkflowNodeService nodeService; + + private static ContextService contextService = ContextServiceImpl.getInstance(); + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + + @Override + public String uploadResourceToBml(String userName, String nodeJson, String inputResourcePath, String projectName) throws IOException { + List resources = nodeParser.getNodeResource(nodeJson); + if (resources != null && resources.size() > 0) { + resources.forEach(resource -> { + if (resource.getVersion() != null && resource.getFileName() != null && resource.getResourceId() != null) { + InputStream resourceInputStream = readResource(userName, resource, inputResourcePath); + Map bmlReturnMap = bmlService.upload(userName, + resourceInputStream, UUID.randomUUID().toString() + ".json", projectName); + resource.setResourceId(bmlReturnMap.get("resourceId").toString()); + resource.setVersion(bmlReturnMap.get("version").toString()); + } else { + logger.warn("Illegal resource information"); + logger.warn("username:{},fileName:{},version:{},resourceId:{}", userName, resource.getFileName(), resource.getVersion(), resource.getResourceId()); + } + }); + } + return nodeParser.updateNodeResource(nodeJson, resources); + } + + private InputStream readResource(String userName, Resource resource, String flowResourcePath) { + String readPath = flowResourcePath + resource.getResourceId() + "_" + resource.getVersion() + ".re"; + return bmlService.readLocalResourceFile(userName, readPath); + } + + @Override + public String uploadAppConnResource(String userName, String projectName, DSSFlow dssFlow, + String nodeJson, String flowContextId, String appConnResourcePath, + Workspace workspace, String orcVersion, List dssLabels) throws DSSErrorException, IOException { + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(nodeJson, Map.class); + String nodeType = nodeJsonMap.get("jobType").toString(); + String nodeId = nodeJsonMap.get("id").toString(); + String nodeResourcePath = appConnResourcePath + File.separator + nodeId + ".appconnre"; + if (nodeType.contains("appjoint")) { + nodeType = nodeType.replace("appjoint", "appconn"); + //兼容0.X导入路径 + nodeResourcePath = nodeResourcePath.replace("appconn", "appjoint"); + } + + Map nodeContent = (LinkedHashMap) nodeJsonMap.get("jobContent"); + CommonAppConnNode appConnNode = new CommonAppConnNode(); + appConnNode.setId(nodeId); + appConnNode.setName((String) nodeJsonMap.get("title")); + appConnNode.setDssLabels(dssLabels); + appConnNode.setNodeType(nodeType); + appConnNode.setJobContent(nodeContent); + appConnNode.setFlowId(dssFlow.getId()); + appConnNode.setProjectId(dssFlow.getProjectID()); + appConnNode.setWorkspace(workspace); + appConnNode.setContextId(flowContextId); + + Map nodeExportContent = null; + logger.info("nodeResourcePath:{}", nodeResourcePath); + File file = new File(nodeResourcePath); + if (file.exists()) { + InputStream resourceInputStream = bmlService.readLocalResourceFile(userName, nodeResourcePath); + Supplier> bmlResourceMap = () -> bmlService.upload(userName, resourceInputStream, UUID.randomUUID().toString() + ".json", + projectName); + Supplier> streamResourceMap = () -> MapUtils.newCommonMap(ImportRequestRef.INPUT_STREAM_KEY, resourceInputStream); + try { + nodeExportContent = nodeService.importNode(userName, appConnNode, bmlResourceMap, streamResourceMap, orcVersion); + } catch (ExternalOperationFailedException e) { + logger.error("failed to import node.", e); + throw new DSSErrorException(e.getErrCode(), e.getMessage()); + } catch (Exception e) { + logger.error("failed to import node.", e); + throw new DSSErrorException(90011, e.getMessage()); + } + if (nodeExportContent != null) { + if (nodeExportContent.get("project_id") != null) { + Long newProjectId = Long.parseLong(nodeExportContent.get("project_id").toString()); + logger.warn(String.format("new appConn node add into dss,dssProjectId: %s,newProjectId: %s", appConnNode.getProjectId(), newProjectId)); + nodeExportContent.remove("project_id"); + } + nodeJsonMap.replace("jobContent", nodeExportContent); + appConnNode.setJobContent(nodeExportContent); + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeJsonMap); + } + } else { + logger.error("appConn node resource file does not exists. nodeId: {}" + nodeId); + } + + return nodeJson; + } + + @Override + public String updateNodeSubflowID(String nodeJson, long subflowID) throws IOException { + return nodeParser.updateSubFlowID(nodeJson, subflowID); + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java new file mode 100644 index 000000000..21bce5d2a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java @@ -0,0 +1,360 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.input.impl; + +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import com.webank.wedatasphere.dss.standard.app.development.utils.DSSJobContentConstant; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import com.webank.wedatasphere.dss.workflow.common.parser.NodeParser; +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.io.input.NodeInputService; +import com.webank.wedatasphere.dss.workflow.io.input.WorkFlowInputService; +import com.webank.wedatasphere.dss.workflow.io.scheduler.NodeImportJob; +import com.webank.wedatasphere.dss.workflow.service.BMLService; +import org.apache.commons.collections.CollectionUtils; +import org.apache.linkis.cs.common.utils.CSCommonUtils; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.workflow.scheduler.DssJobThreadPool.nodeExportThreadPool; + +@Service +public class WorkFlowInputServiceImpl implements WorkFlowInputService { + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private BMLService bmlService; + + @Autowired + private WorkFlowParser workFlowParser; + + @Autowired + private NodeInputService nodeInputService; + + @Autowired + private NodeParser nodeParser; + + @Autowired + private FlowMapper flowMapper; + private static ContextService contextService = ContextServiceImpl.getInstance(); + + @Override + public void inputWorkFlow(String userName, + DSSFlow dssFlow, + String projectName, + String inputProjectPath, + Long parentFlowId, + Workspace workspace, + String orcVersion, + String contextId, + List dssLabels) throws DSSErrorException, IOException { + //todo 不同服务共享导入工程文件包,可采用共享存储 + String flowInputPath = inputProjectPath + File.separator + dssFlow.getName(); + String flowJsonPath = flowInputPath + File.separator + dssFlow.getName() + ".json"; + String flowJson = bmlService.readLocalFlowJsonFile(userName, flowJsonPath); + //如果包含subflow,需要一同导入subflow内容,并更新parrentflow的json内容 + // TODO: 2020/7/31 优化update方法里面的saveContent + String updateFlowJson = updateFlowContextIdAndVersion(userName, + workspace.getWorkspaceName(), + projectName, + flowJson, + dssFlow, + parentFlowId, + contextId, + orcVersion); + updateFlowJson = inputWorkFlowNodes(userName, projectName, updateFlowJson, dssFlow, + flowInputPath, workspace, orcVersion, dssLabels); + List subFlows = dssFlow.getChildren(); + if (subFlows != null) { + for (DSSFlow subFlow : subFlows) { + inputWorkFlow(userName, subFlow, projectName, inputProjectPath, dssFlow.getId(), + workspace, orcVersion, contextId, dssLabels); + } + } + + updateFlowJson = uploadFlowResourceToBml(userName, updateFlowJson, flowInputPath, projectName); + + DSSFlow updateDssFlow = uploadFlowJsonToBml(userName, projectName, dssFlow, updateFlowJson); + //todo add dssflow to database + contextService.checkAndSaveContext(updateFlowJson, String.valueOf(parentFlowId)); + flowMapper.updateFlowInputInfo(updateDssFlow); + + + } + + + private String updateFlowContextIdAndVersion(String userName, + String workspaceName, + String projectName, + String flowJson, + DSSFlow dssFlow, + Long parentFlowId, + String contextId, + String orcVersion) throws IOException, DSSErrorException { + + String parentFlowIdStr = null; + if (parentFlowId != null) { + parentFlowIdStr = parentFlowId.toString(); + } + if (!dssFlow.getRootFlow()) { + contextId = contextService.checkAndInitContext(flowJson, parentFlowIdStr, workspaceName, projectName, dssFlow.getName(), orcVersion, userName); + logger.info("create subflow contextID is " + contextId); + } + return workFlowParser.updateFlowJsonWithMap(flowJson, MapUtils.newCommonMap(CSCommonUtils.CONTEXT_ID_STR, contextId, DSSJobContentConstant.ORC_VERSION_KEY, orcVersion)); + } + + private String inputWorkFlowNodes(String userName, String projectName, String flowJson, + DSSFlow dssFlow, String flowPath, Workspace workspace, + String orcVersion, List dssLabels) throws DSSErrorException, IOException { + List nodeJsonList = workFlowParser.getWorkFlowNodesJson(flowJson); + if (nodeJsonList == null) { + throw new DSSErrorException(90073, "工作流内没有工作流节点,导入失败 " + dssFlow.getName()); + } + String updateContextId = workFlowParser.getValueWithKey(flowJson, CSCommonUtils.CONTEXT_ID_STR); + if (nodeJsonList.size() == 0) { + return flowJson; + } + List subflows = (List) dssFlow.getChildren(); + String workFlowResourceSavePath = flowPath + File.separator + "resource" + File.separator; + String appConnResourceSavePath = flowPath + File.separator + "appconn-resource"; + List> nodeJsonListRes = new ArrayList<>(); + if (nodeJsonList.size() > 0) { + for (String nodeJson : nodeJsonList) { + // TODO: 2020/3/20 暂时注视掉appconn相关 + String updateNodeJson = nodeInputService.uploadResourceToBml(userName, nodeJson, workFlowResourceSavePath, projectName); + updateNodeJson = nodeInputService.uploadAppConnResource(userName, projectName, + dssFlow, updateNodeJson, updateContextId, appConnResourceSavePath, + workspace, orcVersion, dssLabels); + //兼容0.x的key修改 + if(updateNodeJson.contains("wds.linkis.yarnqueue")) { + updateNodeJson = updateNodeJson.replace("wds.linkis.yarnqueue", "wds.linkis.rm.yarnqueue"); + } + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + //更新subflowID + String nodeType = nodeJsonMap.get("jobType").toString(); + if(nodeType.contains("appjoint")){ + nodeJsonMap.replace("jobType",nodeType.replace("appjoint","appconn")); + } + if ("workflow.subflow".equals(nodeType) && CollectionUtils.isNotEmpty(subflows)) { + String subFlowName = nodeJsonMap.get("title").toString(); + logger.info("subflows:{}", subflows); + List DSSFlowList = subflows.stream().filter(subflow -> + subflow.getName().equals(subFlowName) + ).collect(Collectors.toList()); + if (DSSFlowList.size() == 1) { + updateNodeJson = nodeInputService.updateNodeSubflowID(updateNodeJson, DSSFlowList.get(0).getId()); + nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + nodeJsonListRes.add(nodeJsonMap); + } else if (DSSFlowList.size() > 1) { + logger.error("工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + throw new DSSErrorException(90077, "工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + } else { + logger.error("工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + throw new DSSErrorException(90078, "工程内未能找到子工作流节点,导入失败" + subFlowName); + } + } else { + nodeJsonListRes.add(nodeJsonMap); + } + } + } + + return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); + + } + + + private String inputWorkFlowNodes_for_multi_thread(String userName, String projectName, String flowJson, + DSSFlow dssFlow, String flowPath, Workspace workspace, + String orcVersion, List dssLabels) throws DSSErrorException, IOException { + List nodeJsonList = workFlowParser.getWorkFlowNodesJson(flowJson); + if (nodeJsonList == null) { + throw new DSSErrorException(90073, "工作流内没有工作流节点,导入失败 " + dssFlow.getName()); + } + String updateContextId = workFlowParser.getValueWithKey(flowJson, CSCommonUtils.CONTEXT_ID_STR); + if (nodeJsonList.size() == 0) { + return flowJson; + } + List subflows = (List) dssFlow.getChildren(); + String workFlowResourceSavePath = flowPath + File.separator + "resource" + File.separator; + String appConnResourceSavePath = flowPath + File.separator + "appconn-resource"; +// List> nodeJsonListRes = new ArrayList<>(); + List> nodeJsonListRes = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch cdl = new CountDownLatch(nodeJsonList.size()); + AtomicInteger failedCount = new AtomicInteger(0); + if (nodeJsonList.size() > 0) { + for (String nodeJson : nodeJsonList) { + NodeImportJob.ImportJobEntity jobEntity = new NodeImportJob.ImportJobEntity(); + jobEntity.setDssFlow(dssFlow); + jobEntity.setNodeJson(nodeJson); + jobEntity.setUserName(userName); + jobEntity.setProjectName(projectName); + jobEntity.setWorkFlowResourceSavePath(workFlowResourceSavePath); + jobEntity.setAppConnResourceSavePath(appConnResourceSavePath); + jobEntity.setDssLabels(dssLabels); + jobEntity.setWorkspace(workspace); + jobEntity.setUpdateContextId(updateContextId); + jobEntity.setSubflows(subflows); + jobEntity.setOrcVersion(orcVersion); + NodeImportJob nodeImportJob = new NodeImportJob(); + nodeImportJob.setNodeInputService(nodeInputService); + nodeImportJob.setJobEntity(jobEntity); + nodeImportJob.setFailedCount(failedCount); + nodeImportJob.setCountDownLatch(cdl); + nodeImportJob.setNodeJsonListRes(nodeJsonListRes); + nodeExportThreadPool.submit(nodeImportJob); + } + } + // 用户需要等待所有节点导入完成 + boolean success = false; + try { + success = cdl.await(30, TimeUnit.MINUTES); + } catch (InterruptedException e) { + logger.error("failed to import node for workflow:{}", dssFlow.getName(), e); + throw new DSSErrorException(90071, "导入节点超时!"); + } + if (failedCount.get() > 0) { + throw new DSSErrorException(90074, "有节点导入失败,请重试!"); + } + return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); + } + + private String uploadFlowResourceToBml(String userName, String flowJson, String flowResourcePath, String projectName) throws IOException { + + List resourceList = workFlowParser.getWorkFlowResources(flowJson); + //上传文件获取resourceId和version save应该是已经有 + if (resourceList != null) { + resourceList.forEach(resource -> { + InputStream resourceInputStream = readFlowResource(userName, resource, flowResourcePath); + Map bmlReturnMap = bmlService.upload(userName, resourceInputStream, UUID.randomUUID().toString() + ".json", projectName); + resource.setResourceId(bmlReturnMap.get("resourceId").toString()); + resource.setVersion(bmlReturnMap.get("version").toString()); + }); + if (resourceList.size() == 0) { + return flowJson; + } + } + return workFlowParser.updateFlowJsonWithKey(flowJson, "resources", resourceList); + } + + private InputStream readFlowResource(String userName, Resource resource, String flowResourcePath) { + // TODO: 2020/3/20 和导出统一,资源都放resouce 如有问题,后再修改 + String readPath = flowResourcePath + File.separator + "resource" + File.separator + resource.getResourceId() + ".re"; + return bmlService.readLocalResourceFile(userName, readPath); + } + + + public DSSFlow uploadFlowJsonToBml(String userName, String projectName, DSSFlow dssFlow, String flowJson) { + //获取rsourceId,就是jsonPath + Long flowID = dssFlow.getId(); + String resourceId = dssFlow.getResourceId(); + //上传文件获取resourceId和version save应该是已经有 + Map bmlReturnMap; +// if (resourceId != null) { +// bmlReturnMap = bmlService.update(userName, resourceId, flowJson); +// } else { + //上传文件获取resourceId和version save应该是已经有 + bmlReturnMap = bmlService.upload(userName, flowJson, UUID.randomUUID().toString() + ".json", projectName); +// } + + dssFlow.setCreator(userName); + dssFlow.setBmlVersion(bmlReturnMap.get("version").toString()); + dssFlow.setResourceId(bmlReturnMap.get("resourceId").toString()); + dssFlow.setDescription("import update workflow"); + dssFlow.setSource("导入更新"); + + //version表中插入数据 + return dssFlow; + } + + /** + * @param projectId 新环境的projectID + * @param userName + * @param dssFlows + * @return + */ + @Override + public List persistenceFlow(Long projectId, String userName, List dssFlows, + List dssFlowRelations) { + List rootFlows = dssFlows.stream().filter(DSSFlow::getRootFlow).collect(Collectors.toList()); + return rootFlows.stream().map(rf -> setSubFlow(rf, dssFlows, dssFlowRelations, projectId, userName, null)) + .collect(Collectors.toList()); + } + + public DSSFlow setSubFlow(DSSFlow dssFlow, List dssFlows, + List dssFlowRelations, + Long projectId, + String username, DSSFlow parentFlow) { + DSSFlow cyFlow = new DSSFlow(); + BeanUtils.copyProperties(dssFlow, cyFlow, "children", "flowVersions"); + //封装flow信息 + cyFlow.setProjectID(projectId); + cyFlow.setCreator(username); + cyFlow.setCreateTime(new Date()); + cyFlow.setId(null); + flowMapper.insertFlow(cyFlow); + //插入input 关联信息 + + //inputRelationService.insertFlowInputRelation(dssFlow.getId(), sourceEnv, cyFlow.getId()); + + //插入或者更新relation表 + if (parentFlow != null) { + persistenceFlowRelation(cyFlow.getId(), parentFlow.getId()); + if (parentFlow.getChildren() == null) { + parentFlow.setChildren(new ArrayList()); + } + parentFlow.addChildren(cyFlow); + } + List subFlowIds = dssFlowRelations.stream().filter(r -> r.getParentFlowID().equals(dssFlow.getId())).map(DSSFlowRelation::getFlowID).collect(Collectors.toList()); + for (Long subFlowId : subFlowIds) { + DSSFlow subDSSFlow = dssFlows.stream().filter(f -> subFlowId.equals(f.getId())).findFirst().orElse(null); + if (dssFlow.getChildren() == null) { + dssFlow.setChildren(new ArrayList()); + } + dssFlow.addChildren(subDSSFlow); + setSubFlow(subDSSFlow, dssFlows, dssFlowRelations, projectId, username, cyFlow); + } + return cyFlow; + } + + private void persistenceFlowRelation(Long flowID, Long parentFlowID) { + DSSFlowRelation relation = flowMapper.selectFlowRelation(flowID, parentFlowID); + if (relation == null) { + flowMapper.insertFlowRelation(flowID, parentFlowID); + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/IOJobHook.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/IOJobHook.java new file mode 100644 index 000000000..3fd24c50b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/IOJobHook.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.scheduler; + +import com.webank.wedatasphere.dss.workflow.scheduler.DssJob; +import com.webank.wedatasphere.dss.workflow.scheduler.DssJobHook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IOJobHook implements DssJobHook { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Override + public void preExecute(DssJob job) { + + + } + + @Override + public void postExecute(DssJob job) { + + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/IOManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/IOManager.java new file mode 100644 index 000000000..22dd7a715 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/IOManager.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.scheduler; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.workflow.scheduler.DssJobManager; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class IOManager extends DssJobManager { + + @PostConstruct + public void init() { + addDssJobHook(new IOJobHook()); + } + + public void submitIoJob(String userName, String comment, Long... flowIDs) throws DSSErrorException { + PublishJob exportJob = new PublishJob(); + exportJob.setDssJobListener(this); + exportJob.setUser(userName); + exportJob.setFlowIDs(flowIDs); + exportJob.setComment(comment); + submit(exportJob); + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeCopyJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeCopyJob.java new file mode 100644 index 000000000..1b1a6d2d7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeCopyJob.java @@ -0,0 +1,269 @@ +package com.webank.wedatasphere.dss.workflow.io.scheduler; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.dao.NodeInfoMapper; +import com.webank.wedatasphere.dss.workflow.entity.CommonAppConnNode; +import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; +import com.webank.wedatasphere.dss.workflow.io.export.NodeExportService; +import com.webank.wedatasphere.dss.workflow.io.input.NodeInputService; +import com.webank.wedatasphere.dss.workflow.scheduler.DssJob; +import com.webank.wedatasphere.dss.workflow.service.WorkflowNodeService; +import org.apache.linkis.server.BDPJettyServerHelper; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +public class NodeCopyJob extends DssJob { + + private NodeInputService nodeInputService; + + private NodeExportService nodeExportService; + + private NodeInfoMapper nodeInfoMapper; + + private WorkflowNodeService workflowNodeService; + + private JobEntity jobEntity; + + private List> nodeJsonListRes; + + private AtomicInteger failedCount; + + private CountDownLatch countDownLatch; + + @Override + public void run() { + try { + //重新上传一份jar文件到bml + String updateNodeJson = inputNodeFiles(jobEntity.userName, jobEntity.projectName, jobEntity.nodeJson); + + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + //更新subflowID + String nodeType = nodeJsonMap.get("jobType").toString(); + NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(nodeType); + if ("workflow.subflow".equals(nodeType)) { + String subFlowName = nodeJsonMap.get("title").toString(); + List dssFlowList = jobEntity.getSubflows().stream().filter(subflow -> + subflow.getName().equals(subFlowName) + ).collect(Collectors.toList()); + if (dssFlowList.size() == 1) { + updateNodeJson = nodeInputService.updateNodeSubflowID(updateNodeJson, dssFlowList.get(0).getId()); + nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + nodeJsonListRes.add(nodeJsonMap); + } else if (dssFlowList.size() > 1) { + logger.error("工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + throw new DSSErrorException(90077, "工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + } else { + logger.error("工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + throw new DSSErrorException(90078, "工程内未能找到子工作流节点,导入失败" + subFlowName); + } +// } else if (nodeJsonMap.get("jobContent") != null && !((Map) nodeJsonMap.get("jobContent")).containsKey("script")) { + } else if (Boolean.TRUE.equals(nodeInfo.getSupportJump()) && nodeInfo.getJumpType() == 1) { + logger.info("nodeJsonMap.jobContent is:{}", nodeJsonMap.get("jobContent")); + CommonAppConnNode newNode = new CommonAppConnNode(); + CommonAppConnNode oldNode = new CommonAppConnNode(); + oldNode.setJobContent((Map) nodeJsonMap.get("jobContent")); + oldNode.setContextId(jobEntity.getUpdateContextId()); + oldNode.setNodeType(nodeType); + oldNode.setName((String) nodeJsonMap.get("title")); + oldNode.setFlowId(jobEntity.getDssFlow().getId()); + oldNode.setWorkspace(jobEntity.getWorkspace()); + oldNode.setDssLabels(jobEntity.getDssLabels()); + oldNode.setFlowName(jobEntity.getDssFlow().getName()); + oldNode.setProjectId(jobEntity.getDssFlow().getProjectID()); + newNode.setName(oldNode.getName()); + Map jobContent = workflowNodeService.copyNode(jobEntity.userName, newNode, oldNode, jobEntity.orcVersion); + nodeJsonMap.put("jobContent", jobContent); + nodeJsonListRes.add(nodeJsonMap); + } else { + nodeJsonListRes.add(nodeJsonMap); + } + } catch (Exception e) { + failedCount.getAndAdd(1); + logger.error("failed to copy node:", e); + } finally { + countDownLatch.countDown(); + } + } + + //由于每一个节点可能含有jar文件,这个功能不能直接复制使用,因为删掉新版本节点会直接删掉旧版本的node中的jar文件 + //所以重新上传一份jar文件到bml + private String inputNodeFiles(String userName, String projectName, String nodeJson) throws IOException { + String flowPath = IoUtils.generateIOPath(userName, projectName, ""); + String workFlowResourceSavePath = flowPath + File.separator + "resource" + File.separator; + Gson gson = new Gson(); + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(nodeJson).getAsJsonObject(); + DSSNode node = gson.fromJson(jsonObject, new TypeToken() { + }.getType()); + //先导出来 + nodeExportService.downloadNodeResourceToLocal(userName, node, workFlowResourceSavePath); + //后导入到bml + String updateNodeJson = nodeInputService.uploadResourceToBml(userName, nodeJson, workFlowResourceSavePath, projectName); + return updateNodeJson; + } + + public NodeInputService getNodeInputService() { + return nodeInputService; + } + + public void setNodeInputService(NodeInputService nodeInputService) { + this.nodeInputService = nodeInputService; + } + + public JobEntity getJobEntity() { + return jobEntity; + } + + public void setJobEntity(JobEntity jobEntity) { + this.jobEntity = jobEntity; + } + + public List> getNodeJsonListRes() { + return nodeJsonListRes; + } + + public void setNodeJsonListRes(List> nodeJsonListRes) { + this.nodeJsonListRes = nodeJsonListRes; + } + + public CountDownLatch getCountDownLatch() { + return countDownLatch; + } + + public void setCountDownLatch(CountDownLatch countDownLatch) { + this.countDownLatch = countDownLatch; + } + + public AtomicInteger getFailedCount() { + return failedCount; + } + + public void setFailedCount(AtomicInteger failedCount) { + this.failedCount = failedCount; + } + + public NodeExportService getNodeExportService() { + return nodeExportService; + } + + public void setNodeExportService(NodeExportService nodeExportService) { + this.nodeExportService = nodeExportService; + } + + public NodeInfoMapper getNodeInfoMapper() { + return nodeInfoMapper; + } + + public void setNodeInfoMapper(NodeInfoMapper nodeInfoMapper) { + this.nodeInfoMapper = nodeInfoMapper; + } + + public WorkflowNodeService getWorkflowNodeService() { + return workflowNodeService; + } + + public void setWorkflowNodeService(WorkflowNodeService workflowNodeService) { + this.workflowNodeService = workflowNodeService; + } + + + public static class JobEntity { + private String userName; + private String projectName; + private DSSFlow dssFlow; + private List subflows; + private Workspace workspace; + private String orcVersion; + private List dssLabels; + private String nodeJson; + private String updateContextId; + + public String getUpdateContextId() { + return updateContextId; + } + + public void setUpdateContextId(String updateContextId) { + this.updateContextId = updateContextId; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public DSSFlow getDssFlow() { + return dssFlow; + } + + public void setDssFlow(DSSFlow dssFlow) { + this.dssFlow = dssFlow; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public String getOrcVersion() { + return orcVersion; + } + + public void setOrcVersion(String orcVersion) { + this.orcVersion = orcVersion; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public String getNodeJson() { + return nodeJson; + } + + public void setNodeJson(String nodeJson) { + this.nodeJson = nodeJson; + } + + public List getSubflows() { + return subflows; + } + + public void setSubflows(List subflows) { + this.subflows = subflows; + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeExportJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeExportJob.java new file mode 100644 index 000000000..6534d29f0 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeExportJob.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.dss.workflow.io.scheduler; + +public class NodeExportJob { +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeImportJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeImportJob.java new file mode 100644 index 000000000..7524db301 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/NodeImportJob.java @@ -0,0 +1,220 @@ +package com.webank.wedatasphere.dss.workflow.io.scheduler; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.io.input.NodeInputService; +import com.webank.wedatasphere.dss.workflow.scheduler.DssJob; +import org.apache.commons.collections.CollectionUtils; +import org.apache.linkis.server.BDPJettyServerHelper; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +public class NodeImportJob extends DssJob { + + private NodeInputService nodeInputService; + + private ImportJobEntity jobEntity; + + private List> nodeJsonListRes; + + private AtomicInteger failedCount; + + private CountDownLatch countDownLatch; + + @Override + public void run() { + try { + // TODO: 2020/3/20 暂时注视掉appconn相关 + String updateNodeJson = nodeInputService.uploadResourceToBml(jobEntity.getUserName(), jobEntity.getNodeJson(), + jobEntity.getWorkFlowResourceSavePath(), jobEntity.getProjectName()); + updateNodeJson = nodeInputService.uploadAppConnResource(jobEntity.getUserName(), jobEntity.getProjectName(), + jobEntity.getDssFlow(), updateNodeJson, jobEntity.getUpdateContextId(), jobEntity.getAppConnResourceSavePath(), + jobEntity.getWorkspace(), jobEntity.getOrcVersion(), jobEntity.getDssLabels()); + //兼容0.x的key修改 + if (updateNodeJson.contains("wds.linkis.yarnqueue")) { + updateNodeJson = updateNodeJson.replace("wds.linkis.yarnqueue", "wds.linkis.rm.yarnqueue"); + } + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + //更新subflowID + String nodeType = nodeJsonMap.get("jobType").toString(); + if (nodeType.contains("appjoint")) { + nodeJsonMap.replace("jobType", nodeType.replace("appjoint", "appconn")); + } + if ("workflow.subflow".equals(nodeType) && CollectionUtils.isNotEmpty(jobEntity.getSubflows())) { + String subFlowName = nodeJsonMap.get("title").toString(); + logger.info("subflows:{}", jobEntity.getSubflows()); + List dssFlowList = jobEntity.getSubflows().stream().filter(subflow -> subflow.getName().equals(subFlowName)).collect(Collectors.toList()); + if (dssFlowList.size() == 1) { + updateNodeJson = nodeInputService.updateNodeSubflowID(updateNodeJson, dssFlowList.get(0).getId()); + nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + nodeJsonListRes.add(nodeJsonMap); + } else if (dssFlowList.size() > 1) { + logger.error("工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + throw new DSSErrorException(90077, "工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + } else { + logger.error("工程内未能找到子工作流节点,导入失败" + subFlowName); + throw new DSSErrorException(90078, "工程内未能找到子工作流节点,导入失败" + subFlowName); + } + } else { + nodeJsonListRes.add(nodeJsonMap); + } +// countDownLatch.countDown(); + } catch (Exception e) { + //todo 失败重试 + failedCount.getAndAdd(1); + logger.error("failed to import node:", e); + }finally { + countDownLatch.countDown(); + } + + } + + public NodeInputService getNodeInputService() { + return nodeInputService; + } + + public void setNodeInputService(NodeInputService nodeInputService) { + this.nodeInputService = nodeInputService; + } + + public ImportJobEntity getJobEntity() { + return jobEntity; + } + + public void setJobEntity(ImportJobEntity jobEntity) { + this.jobEntity = jobEntity; + } + + public List> getNodeJsonListRes() { + return nodeJsonListRes; + } + + public void setNodeJsonListRes(List> nodeJsonListRes) { + this.nodeJsonListRes = nodeJsonListRes; + } + + public CountDownLatch getCountDownLatch() { + return countDownLatch; + } + + public void setCountDownLatch(CountDownLatch countDownLatch) { + this.countDownLatch = countDownLatch; + } + + public AtomicInteger getFailedCount() { + return failedCount; + } + + public void setFailedCount(AtomicInteger failedCount) { + this.failedCount = failedCount; + } + + + public static class ImportJobEntity { + private String userName; + private String projectName; + private DSSFlow dssFlow; + private List subflows; + private String workFlowResourceSavePath; + private String appConnResourceSavePath; + private Workspace workspace; + private String orcVersion; + private List dssLabels; + private String nodeJson; + private String updateContextId; + + public String getUpdateContextId() { + return updateContextId; + } + + public void setUpdateContextId(String updateContextId) { + this.updateContextId = updateContextId; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public DSSFlow getDssFlow() { + return dssFlow; + } + + public void setDssFlow(DSSFlow dssFlow) { + this.dssFlow = dssFlow; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public String getOrcVersion() { + return orcVersion; + } + + public void setOrcVersion(String orcVersion) { + this.orcVersion = orcVersion; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public String getNodeJson() { + return nodeJson; + } + + public void setNodeJson(String nodeJson) { + this.nodeJson = nodeJson; + } + + public String getWorkFlowResourceSavePath() { + return workFlowResourceSavePath; + } + + public void setWorkFlowResourceSavePath(String workFlowResourceSavePath) { + this.workFlowResourceSavePath = workFlowResourceSavePath; + } + + public String getAppConnResourceSavePath() { + return appConnResourceSavePath; + } + + public void setAppConnResourceSavePath(String appConnResourceSavePath) { + this.appConnResourceSavePath = appConnResourceSavePath; + } + + public List getSubflows() { + return subflows; + } + + public void setSubflows(List subflows) { + this.subflows = subflows; + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/PublishJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/PublishJob.java new file mode 100644 index 000000000..86173edd7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/scheduler/PublishJob.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.io.scheduler; + +import com.webank.wedatasphere.dss.workflow.scheduler.DssJob; + +public class PublishJob extends DssJob { + + + + private String comment; + + private Long[] flowIDs; + + + @Override + public void run() { + try { + dssJobListener.onJobRunning(this, "导出job正在运行:" + id); + //todo 工作流发布逻辑 + + dssJobListener.onJobSucceed(this, "导出job执行成功:" + id); + } catch (Throwable e) { + logger.error("导出job执行失败,原因:", e); + Throwable cause = e.getCause(); + String errorMsg = cause == null ? e.getMessage() : cause.getMessage(); + dssJobListener.onJobFailed(this, "导出job执行失败:" + errorMsg); + } + } + + + + public Long[] getFlowIDs() { + return flowIDs; + } + + public void setFlowIDs(Long[] flowIDs) { + this.flowIDs = flowIDs; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java new file mode 100644 index 000000000..2789d936f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java @@ -0,0 +1,276 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.lock; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; +import org.apache.linkis.DataWorkCloudApplication; +import org.apache.linkis.common.utils.Utils; +import org.apache.commons.lang.StringUtils; +import org.apache.poi.util.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DuplicateKeyException; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; + + +/** + * 工作流编辑分布式锁 + */ +public class DSSFlowEditLockManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSFlowEditLockManager.class); + + private volatile static boolean isInit; + + private static LockMapper lockMapper; + + + private static final DelayQueue unLockEvents = new DelayQueue<>(); + private static final ThreadLocal sdf = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); + + protected DSSFlowEditLockManager() { + } + + static { + LOGGER.info("unLockEvents移除定时线程开启..."); + LOGGER.info("编辑锁超时时间为:{} ms", DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue()); + init(); + //程序重启时,删除所有编辑锁 + lockMapper.deleteALL(); + Utils.defaultScheduler().scheduleAtFixedRate(() -> { + try { + UnLockEvent pop = unLockEvents.poll(); + if (pop != null) { + DSSFlowEditLock flowEditLock = pop.getFlowEditLock(); + long flowId = flowEditLock.getFlowID(); + DSSFlowEditLock updateFlowEditLock = lockMapper.getFlowEditLockByID(flowId); + //队列对象如果过期,先去数据库查询锁,并判断锁是否过期 + if (updateFlowEditLock != null && isLockExpire(updateFlowEditLock)) { + //锁过期,移除记录 + String expireTime = sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())); + LOGGER.info("删除用户{}的编辑锁,flowId:{},lockContent:{},owner:{},update_time:{},expireTime:{}", + updateFlowEditLock.getUsername(), flowId, updateFlowEditLock.getLockContent(), + updateFlowEditLock.getOwner(), updateFlowEditLock.getUpdateTime(), expireTime); + lockMapper.clearExpire(expireTime, flowId); + } else if (updateFlowEditLock == null) { + LOGGER.info("lock already clear"); + } else { + //锁没有过期,延长队列时间 + pop.setExpireTime(updateFlowEditLock.getUpdateTime().getTime() + DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue()); + unLockEvents.offer(pop); + } + } + } catch (Exception e) { + LOGGER.error("定时清除工作流编辑锁错误", e); + } + }, 0, 1, TimeUnit.SECONDS); + } + + /** + * 获取工作流编辑锁 + * + * @param dssFlow 工作流对象 + * @param username 登陆用户 + * @param owner 锁的拥有者(ticketId) + * @return 锁id + * @throws DSSErrorException ex + */ + public static String tryAcquireLock(DSSFlow dssFlow, String username, String owner) throws DSSErrorException { + if (StringUtils.isBlank(username)) { + throw new DSSErrorException(60061, "tryAcquireLock failed , because username is null"); + } + if (StringUtils.isBlank(owner)) { + throw new DSSErrorException(60062, "tryAcquireLock failed , because owner is null"); + } + Long flowID = dssFlow.getId(); + if (flowID == null) { + throw new DSSErrorException(60063, "tryAcquireLock failed , because flowId is null"); + } + String lock; + init(); + DSSFlowEditLock flowEditLock = lockMapper.getPersonalFlowEditLock(flowID, owner); + + if (flowEditLock != null && isLockExpire(flowEditLock)) { + // 1.另外的dss-server服务挂掉了(main);2.记录已经过期,但是UnLockEvent 尚未(即将)去更新数据库;3.记录已经过期,但是updateTime 尚未(即将)更新数据库 + flowEditLock.setExpire(true); + int i = lockMapper.compareAndSwap(flowEditLock); + LOGGER.info("try to set lock to an expire state,row:{},lock:{}", i, flowEditLock); + if (i == 1) { + //更新成功,尝试获取新锁 + LOGGER.info("unlock success,lock:{}", flowEditLock); + lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), flowID); + lock = generateLock(flowID, username, owner); + } else { + // 失败的缘由: 1.另外的dss-server也走到了这步,优先更新了;2.UnLockEvent;3.用户刚好点了saveFlow延续 + throw new DSSErrorException(60055, "acquire lock failed"); + } + } else if (flowEditLock != null) { + lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), flowID); + lock = generateLock(flowID, username, owner); + } else { + // 插入锁,获取到数据库自增id + lock = generateLock(flowID, username, owner); + } + // 插入成功,返回lock,push一条记录到queue中,插入失败,返回null + return lock; + } + + private static String generateLock(Long flowID, String username, String owner) throws DSSErrorException { + try { + String lockContent = UUID.randomUUID().toString(); + DSSFlowEditLock newLock = new DSSFlowEditLock(); + newLock.setExpire(false); + Date date = new Date(); + newLock.setCreateTime(date); + newLock.setUpdateTime(date); + newLock.setFlowID(flowID); + newLock.setLockContent(lockContent); + newLock.setOwner(owner); + newLock.setUsername(username); + lockMapper.insertLock(newLock); + //推到queue中 + UnLockEvent unLockEvent = new UnLockEvent(); + unLockEvent.setCreateTime(date.getTime()); + unLockEvent.setExpireTime(date.getTime() + DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue()); + unLockEvent.setFlowEditLock(newLock); + unLockEvents.offer(unLockEvent); + return lockContent; + } catch (DuplicateKeyException e) { + LOGGER.warn("acquire lock failed", e); + DSSFlowEditLock personalFlowEditLock = lockMapper.getPersonalFlowEditLock(flowID, null); + String userName = Optional.ofNullable(personalFlowEditLock).map(DSSFlowEditLock::getUsername).orElse(null); + throw new DSSErrorException(DSSWorkFlowConstant.EDIT_LOCK_ERROR_CODE, "用户" + userName + "已锁定编辑"); + } + } + + public static void deleteLock(String flowEditLock) throws DSSErrorException { + try { + if (StringUtils.isNotBlank(flowEditLock)) { + DSSFlowEditLock dssFlowEditLock = lockMapper.getFlowEditLockByLockContent(flowEditLock); + if (dssFlowEditLock != null) { + lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), dssFlowEditLock.getFlowID()); + } + } + } catch (Exception e) { + LOGGER.error("flowEditLock delete failed,flowId:{}", flowEditLock, e); + throw new DSSErrorException(60059, "工作流编辑锁主动释放失败,flowId:" + flowEditLock + ""); + } + } + + public static String updateLock(String lock) throws DSSErrorException { + if (StringUtils.isBlank(lock)) { + throw new DSSErrorException(60066, "update workflow failed because you do not have flowEditLock!"); + } + //保存并刷新数据库更新时间 + DSSFlowEditLock dssFlowEditLock = new DSSFlowEditLock(); + dssFlowEditLock.setUpdateTime(new Date()); + dssFlowEditLock.setLockContent(lock); + DSSFlowEditLock updateFlowEditLock = lockMapper.getFlowEditLockByLockContent(lock); + if (updateFlowEditLock == null || updateFlowEditLock.getExpire()) { + lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), 0L); + throw new DSSErrorException(60057, "编辑锁已过期,请刷新页面"); + } + try { + lockMapper.compareAndSwap(dssFlowEditLock); + return lock; + } catch (Exception e) { + LOGGER.error("unexpected error occurred when update dss flow edit lock,{}", dssFlowEditLock, e); + lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), updateFlowEditLock.getFlowID()); + throw new DSSErrorException(60059, "工作流编辑锁更新出错,请刷新页面"); + } + } + + public static boolean isLockExpire(DSSFlowEditLock flowEditLock) { + return System.currentTimeMillis() - flowEditLock.getUpdateTime().getTime() >= DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue(); + } + + public static void init() { + if (!isInit) { + synchronized (DSSFlowEditLockManager.class) { + if (!isInit) { + lockMapper = DataWorkCloudApplication.getApplicationContext().getBean(LockMapper.class); + isInit = true; + } + } + } + } + + public static class UnLockEvent implements Delayed { + private long createTime; + private long expireTime; + private DSSFlowEditLock flowEditLock; + + DSSFlowEditLock getFlowEditLock() { + return flowEditLock; + } + + void setFlowEditLock(DSSFlowEditLock flowEditLock) { + this.flowEditLock = flowEditLock; + } + + public boolean isExpire() { + return System.currentTimeMillis() - expireTime > 0; + } + + public long getCreateTime() { + return createTime; + } + + void setCreateTime(long createTime) { + this.createTime = createTime; + } + + public long getExpireTime() { + return this.expireTime; + } + + void setExpireTime(long expireTime) { + this.expireTime = expireTime; + } + + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(expireTime - System.currentTimeMillis(), unit); + } + + @Override + public int compareTo(Delayed o) { + return (int) (getDelay(TimeUnit.MILLISECONDS) - o.getDelay(TimeUnit.MILLISECONDS)); + } + + @Override + public String toString() { + return "UnLockEvent{" + + "createTime=" + createTime + + ", expireTime=" + expireTime + + ", flowEditLock=" + flowEditLock + + '}'; + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/Lock.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/Lock.java new file mode 100644 index 000000000..a6b4bb802 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/Lock.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.lock; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface Lock { + String type() default LockEnum.EQUAL; +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/LockEnum.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/LockEnum.java new file mode 100644 index 000000000..2dc310f10 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/LockEnum.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.lock; + +public interface LockEnum { + String ADD = "+"; + String EQUAL = "="; +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java new file mode 100644 index 000000000..bcaf0dce7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.parser; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.workflow.common.parser.NodeParser; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class DefaultNodeParser implements NodeParser { + @Override + public String updateNodeResource(String nodeJson, List resources) throws IOException { + if(resources.size() ==0){ + return nodeJson; + } + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(nodeJson, Map.class); + nodeJsonMap.replace("resources",resources); + + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeJsonMap); + } + + @Override + public String updateNodeJobContent(String nodeJson, Map nodeExportContent) throws IOException { + if(nodeExportContent ==null){ + return nodeJson; + } + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(nodeJson, Map.class); + nodeJsonMap.replace("jobContent",nodeExportContent); + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeJsonMap); + } + + @Override + public String updateSubFlowID(String nodeJson, long subflowId) throws IOException { + Map nodeContentMap = new HashMap<>(); + nodeContentMap.put("embeddedFlowId",subflowId); + return updateNodeJobContent(nodeJson,nodeContentMap); + } + + @Override + public String getNodeValue(String key, String nodeJson) throws IOException { + if(StringUtils.isEmpty(nodeJson)){ + return null; + } + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(nodeJson, Map.class); + return nodeJsonMap.get(key).toString(); + + } + + @Override + public List getNodeResource(String nodeJson) { + Gson gson = new Gson(); + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(nodeJson).getAsJsonObject(); + JsonArray resourcesJsonArray = jsonObject.getAsJsonArray("resources"); + List resources = gson.fromJson(resourcesJsonArray, new com.google.gson.reflect.TypeToken>() { + }.getType()); + return resources; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultWorkFlowParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultWorkFlowParser.java new file mode 100644 index 000000000..b6d2451ae --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultWorkFlowParser.java @@ -0,0 +1,122 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.parser; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.reflect.TypeToken; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; +import com.webank.wedatasphere.dss.common.entity.node.DSSEdgeDefault; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Component +public class DefaultWorkFlowParser implements WorkFlowParser { + @Override + public List getWorkFlowResources(String workFlowJson) { + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(workFlowJson).getAsJsonObject(); + JsonArray resourcesJsonArray = jsonObject.getAsJsonArray("resources"); + List resources = DSSCommonUtils.COMMON_GSON.fromJson(resourcesJsonArray, new com.google.gson.reflect.TypeToken>() { + }.getType()); + return resources; + } + + @Override + public List getWorkFlowNodes(String workFlowJson) { + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(workFlowJson).getAsJsonObject(); + JsonArray nodeJsonArray = jsonObject.getAsJsonArray("nodes"); + List dwsNodes = DSSCommonUtils.COMMON_GSON.fromJson(nodeJsonArray, new TypeToken>() { + }.getType()); + return dwsNodes; + } + + @Override + public List getWorkFlowEdges(String workFlowJson) { + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(workFlowJson).getAsJsonObject(); + JsonArray edgeJsonArray = jsonObject.getAsJsonArray("edges"); + List edges = DSSCommonUtils.COMMON_GSON.fromJson(edgeJsonArray, new TypeToken>() { + }.getType()); + return edges; + } + + @Override + public List getWorkFlowNodesJson(String workFlowJson) { + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(workFlowJson).getAsJsonObject(); + JsonArray nodeJsonArray = jsonObject.getAsJsonArray("nodes"); + if (nodeJsonArray == null) { + return null; + } + List nodeJsonList = DSSCommonUtils.COMMON_GSON.fromJson(nodeJsonArray.toString(), new TypeToken>() { + }.getType()); + return nodeJsonList.stream().map(DSSCommonUtils.COMMON_GSON::toJson).collect(Collectors.toList()); + } + + @Override + public String updateFlowJsonWithKey(String workFlowJson, String key, Object value) throws IOException { + if (value == null || key == null) { + return workFlowJson; + } + Map flowJsonObject = BDPJettyServerHelper.jacksonJson().readValue(workFlowJson, Map.class); + flowJsonObject.replace(key, value); + String updatedJson = BDPJettyServerHelper.jacksonJson().writeValueAsString(flowJsonObject); + return updatedJson; + } + + @Override + public String updateFlowJsonWithMap(String workFlowJson, Map props) throws JsonProcessingException { + if (MapUtils.isEmpty(props)) { + return workFlowJson; + } + Map flowJsonObject = BDPJettyServerHelper.jacksonJson().readValue(workFlowJson, Map.class); + props.forEach(flowJsonObject::replace); + return BDPJettyServerHelper.jacksonJson().writeValueAsString(flowJsonObject); + } + + @Override + public String getValueWithKey(String workFlowJson, String key) throws IOException { + if (key == null) { + return null; + } + Map flowJsonObject = BDPJettyServerHelper.jacksonJson().readValue(workFlowJson, Map.class); + + Object value = flowJsonObject.get(key); + if (value == null) { + return null; + } else if (value instanceof String) { + return (String) value; + } else { + return value.toString(); + } + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/ContextServiceRestful.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/ContextServiceRestful.java new file mode 100644 index 000000000..b4e322a87 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/ContextServiceRestful.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.restful; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.workflow.cs.service.CSTableService; +import com.webank.wedatasphere.dss.workflow.entity.request.QueryTableMetaRequest; +import com.webank.wedatasphere.dss.workflow.entity.request.TablesRequest; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; + +@RequestMapping(path = "/dss/workflow", produces = {"application/json"}) +@RestController +public class ContextServiceRestful { + + @Autowired + private CSTableService csTableService; + + @RequestMapping(value = "tables",method = RequestMethod.POST) + public Message tables(HttpServletRequest req, @RequestBody TablesRequest tablesRequest) throws DSSErrorException { + String userName = SecurityFilter.getLoginUsername(req); + String contextIDStr = tablesRequest.getContextID(); + String nodeName = tablesRequest.getNodeName(); + return Message.ok().data("tables", csTableService.queryTables("default", contextIDStr, nodeName)); + } + + @RequestMapping(value = "columns",method = RequestMethod.POST) + public Message queryTableMeta(HttpServletRequest req,@RequestBody QueryTableMetaRequest queryTableMetaRequest) throws DSSErrorException { + String userName = SecurityFilter.getLoginUsername(req); + String contextIDStr = queryTableMetaRequest.getContextID(); + String contextKeyStr = queryTableMetaRequest.getContextKey(); + return Message.ok().data("columns", csTableService.queryTableMeta("default", contextIDStr, contextKeyStr)); + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java new file mode 100644 index 000000000..6ce40ccfb --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java @@ -0,0 +1,305 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.restful; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.dss.appconn.manager.utils.AppConnManagerUtils; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import com.webank.wedatasphere.dss.workflow.WorkFlowManager; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.entity.request.*; +import com.webank.wedatasphere.dss.workflow.entity.vo.ExtraToolBarsVO; +import com.webank.wedatasphere.dss.workflow.exception.DSSWorkflowErrorException; +import com.webank.wedatasphere.dss.workflow.lock.DSSFlowEditLockManager; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import com.webank.wedatasphere.dss.workflow.service.PublishService; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.PostConstruct; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.validation.constraints.NotNull; +import java.io.IOException; +import java.util.*; + +@RestController +@RequestMapping(path = "/dss/workflow", produces = {"application/json"}) +public class FlowRestfulApi { + private static final Logger LOGGER = LoggerFactory.getLogger(FlowRestfulApi.class); + + private ContextService contextService = ContextServiceImpl.getInstance(); + @Autowired + private DSSFlowService flowService; + @Autowired + private PublishService publishService; + @Autowired + private DSSFlowService dssFlowService; + @Autowired + private WorkFlowManager workFlowManager; + + @PostConstruct + public void init() { + AppConnManagerUtils.autoLoadAppConnManager(); + } + + /** + * 添加subflow节点 + * + * @param req + * @param addFlowRequest + * @return + * @throws DSSErrorException + * @throws JsonProcessingException + */ + @RequestMapping(value = "addFlow", method = RequestMethod.POST) + public Message addFlow(HttpServletRequest req, @RequestBody AddFlowRequest addFlowRequest) throws DSSErrorException, JsonProcessingException { + //如果是子工作流,那么分类应该是和父类一起的? + String userName = SecurityFilter.getLoginUsername(req); + // TODO: 2019/5/23 flowName工程名下唯一校验 + String name = addFlowRequest.getName(); + String workspaceName = addFlowRequest.getWorkspaceName(); + String projectName = addFlowRequest.getProjectName(); + String version = addFlowRequest.getVersion(); + String description = addFlowRequest.getDescription(); + Long parentFlowID = addFlowRequest.getParentFlowID(); + String uses = addFlowRequest.getUses(); + List dssLabelList = new ArrayList<>(); + dssLabelList.add(new EnvDSSLabel(addFlowRequest.getLabels().getRoute())); + String contextId = contextService.createContextID(workspaceName, projectName, name, version, userName); + DSSFlow dssFlow = workFlowManager.createWorkflow(userName, null, name, contextId, description, parentFlowID, + uses, new ArrayList<>(), dssLabelList, null, null); + + // TODO: 2019/5/16 空值校验,重复名校验 + return Message.ok().data("flow", dssFlow); + } + + @RequestMapping(value = "publishWorkflow", method = RequestMethod.POST) + public Message publishWorkflow(HttpServletRequest request, @RequestBody PublishWorkflowRequest publishWorkflowRequest) throws Exception { + Long workflowId = publishWorkflowRequest.getWorkflowId(); + Map labels = new HashMap<>(); + labels.put(EnvDSSLabel.DSS_ENV_LABEL_KEY, publishWorkflowRequest.getLabels().getRoute()); + String comment = publishWorkflowRequest.getComment(); + Workspace workspace = SSOHelper.getWorkspace(request); + String publishUser = SecurityFilter.getLoginUsername(request); + Message message; + try { + String taskId = publishService.submitPublish(publishUser, workflowId, labels, workspace, comment); + LOGGER.info("submit publish task ok ,taskId is {}.", taskId); + if (DSSWorkFlowConstant.PUBLISHING_ERROR_CODE.equals(taskId)) { + message = Message.error("发布工程已经含有工作流,正在发布中,请稍后再试"); + } else if (StringUtils.isNotEmpty(taskId)) { + message = Message.ok("生成工作流发布任务成功").data("releaseTaskId", taskId); + } else { + LOGGER.error("taskId {} is error.", taskId); + message = Message.error("发布工作流失败"); + } + } catch (DSSWorkflowErrorException e) { + throw e; + } catch (final Throwable t) { + LOGGER.error("failed to submit publish task for workflow id {}.", workflowId, t); + message = Message.error("发布工作流失败"); + } + return message; + } + + /** + * 获取发布任务状态 + * + * @param request + * @param releaseTaskId + * @return + */ + @RequestMapping(value = "getReleaseStatus", method = RequestMethod.GET) + public Message getReleaseStatus(HttpServletRequest request, + @NotNull(message = "查询的发布id不能为空") @RequestParam(required = false, name = "releaseTaskId") Long releaseTaskId) { + String username = SecurityFilter.getLoginUsername(request); + Message message; + try { + ResponseConvertOrchestrator response = publishService.getStatus(username, releaseTaskId.toString()); + if (null != response.getResponse()) { + String status = response.getResponse().getJobStatus().toString(); + status = StringUtils.isNotBlank(status) ? status.toLowerCase() : status; + //将发布失败原因,返回前端 + if ("failed".equalsIgnoreCase(status)) { + message = Message.error("发布失败:" + response.getResponse().getMessage()).data("status", status); + } else if (StringUtils.isNotBlank(status)) { + message = Message.ok("获取进度成功").data("status", status); + } else { + LOGGER.error("status is null or empty, failed to get status"); + message = Message.error("获取进度失败"); + } + } else { + LOGGER.error("status is null or empty, failed to get status"); + message = Message.error("获取进度失败"); + } + } catch (final Throwable t) { + LOGGER.error("Failed to get release status for {}", releaseTaskId, t); + message = Message.error("发布异常:" + t.getMessage()); + } + return message; + } + + + /** + * 更新工作流的基本信息,不包括更新Json,BML版本等 + * + * @param req + * @param updateFlowBaseInfoRequest + * @return + * @throws DSSErrorException + */ + + @RequestMapping(value = "updateFlowBaseInfo", method = RequestMethod.POST) + public Message updateFlowBaseInfo(HttpServletRequest req, @RequestBody UpdateFlowBaseInfoRequest updateFlowBaseInfoRequest) throws DSSErrorException { + Long flowID = updateFlowBaseInfoRequest.getId(); + String name = updateFlowBaseInfoRequest.getName(); + String description = updateFlowBaseInfoRequest.getDescription(); + String uses = updateFlowBaseInfoRequest.getUses(); + // TODO: 2019/6/13 projectVersionID的更新校验 + //这里可以不做事务 + DSSFlow dssFlow = new DSSFlow(); + dssFlow.setId(flowID); + dssFlow.setName(name); + dssFlow.setDescription(description); + dssFlow.setUses(uses); + flowService.updateFlowBaseInfo(dssFlow); + return Message.ok(); + } + + /** + * 读取工作流的Json数据,提供给前端渲染 + * + * @param req + * @param flowID + * @return + * @throws DSSErrorException + */ + + @RequestMapping(value = "get", method = RequestMethod.GET) + public Message get(HttpServletRequest req, + @RequestParam(required = false, name = "flowId") Long flowID, + @RequestParam(required = false, name = "isNotHaveLock") Boolean isNotHaveLock) throws DSSErrorException { + String username = SecurityFilter.getLoginUsername(req); + DSSFlow dssFlow = flowService.getFlow(flowID); + if (isNotHaveLock != null && isNotHaveLock) { + return Message.ok().data("flow", dssFlow); + } + Cookie[] cookies = req.getCookies(); + String ticketId = Arrays.stream(cookies).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())).findFirst().map(Cookie::getValue).get(); + if (dssFlow != null) { + // 尝试获取工作流编辑锁 + try { + String flowEditLock = DSSFlowEditLockManager.tryAcquireLock(dssFlow, username, ticketId); + dssFlow.setFlowEditLock(flowEditLock); + } catch (DSSErrorException e) { + if (DSSWorkFlowConstant.EDIT_LOCK_ERROR_CODE == e.getErrCode()) { + return Message.error(e.getDesc()); + } + throw e; + } + } + return Message.ok().data("flow", dssFlow); + } + + @RequestMapping(value = "deleteFlow", method = RequestMethod.POST) + public Message deleteFlow(HttpServletRequest req, @RequestBody DeleteFlowRequest deleteFlowRequest) throws DSSErrorException { + Long flowID = deleteFlowRequest.getId(); + boolean sure = deleteFlowRequest.getSure() != null && deleteFlowRequest.getSure().booleanValue(); + // TODO: 2019/6/13 projectVersionID的更新校验 + //state为true代表曾经发布过 + if (flowService.getFlowByID(flowID).getState() && !sure) { + return Message.ok().data("warmMsg", "该工作流曾经发布过,删除将会将该工作流的所有版本都删除,是否继续?"); + } + flowService.batchDeleteFlow(Arrays.asList(flowID)); + return Message.ok(); + } + + /** + * 工作流保存接口,如工作流Json内容有变化,会更新工作流的Json内容 + * + * @param req + * @param saveFlowRequest + * @return + * @throws DSSErrorException + * @throws IOException + */ + @RequestMapping(value = "saveFlow", method = RequestMethod.POST) +// @ProjectPrivChecker + public Message saveFlow(HttpServletRequest req, @RequestBody SaveFlowRequest saveFlowRequest) throws DSSErrorException, IOException { + Long flowID = saveFlowRequest.getId(); + String jsonFlow = saveFlowRequest.getJson(); + String workspaceName = saveFlowRequest.getWorkspaceName(); + String projectName = saveFlowRequest.getProjectName(); + + Boolean isNotHaveLock = saveFlowRequest.getNotHaveLock(); + String userName = SecurityFilter.getLoginUsername(req); + String version; + synchronized (DSSWorkFlowConstant.saveFlowLock.intern(flowID)) { + if (isNotHaveLock != null && isNotHaveLock.booleanValue()) { + version = flowService.saveFlow(flowID, jsonFlow, null, userName, workspaceName, projectName); + return Message.ok().data("flowVersion", version).data("flowEditLock", null); + } + version = flowService.saveFlow(flowID, jsonFlow, null, userName, workspaceName, projectName); + } + return Message.ok().data("flowVersion", version); + } + + /** + * 工作流编辑锁更新接口 + * + * @param req request + * @param flowEditLock 老的编辑锁 + * @return 新的编辑锁 + */ + @RequestMapping(value = "/updateFlowEditLock", method = RequestMethod.GET) + public Message updateFlowEditLock(HttpServletRequest req, @RequestParam(required = false, name = "flowEditLock") String flowEditLock) throws DSSErrorException { + if (StringUtils.isBlank(flowEditLock)) { + throw new DSSErrorException(60067, "update flowEditLock failed,because flowEditLock is empty"); + } + return Message.ok().data("flowEditLock", DSSFlowEditLockManager.updateLock(flowEditLock)); + } + + @RequestMapping(value = "/getExtraToolBars", method = RequestMethod.POST) + public Message getExtraToolBars(HttpServletRequest req, @RequestBody GetExtraToolBarsRequest getExtraToolBarsRequest) throws DSSErrorException { + String userName = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + List barsVOList = flowService.getExtraToolBars(workspace.getWorkspaceId(), getExtraToolBarsRequest.getProjectId()); + return Message.ok().data("extraBars", barsVOList); + } + + + @RequestMapping(value = "/deleteFlowEditLock/{flowEditLock}", method = RequestMethod.POST) + public Message deleteFlowEditLock(HttpServletRequest req, @PathVariable("flowEditLock") String flowEditLock) throws DSSErrorException { + DSSFlowEditLockManager.deleteLock(flowEditLock); + return Message.ok(); + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/NodeRestfulApi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/NodeRestfulApi.java new file mode 100644 index 000000000..e634be52c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/NodeRestfulApi.java @@ -0,0 +1,302 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.restful; + +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.standard.app.development.utils.DSSJobContentConstant; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import com.webank.wedatasphere.dss.workflow.cs.DSSCSHelper; +import com.webank.wedatasphere.dss.workflow.entity.*; +import com.webank.wedatasphere.dss.workflow.entity.request.AppConnNodeUrlRequest; +import com.webank.wedatasphere.dss.workflow.entity.request.BatchDeleteAppConnNodeRequest; +import com.webank.wedatasphere.dss.workflow.entity.request.CreateExternalNodeRequest; +import com.webank.wedatasphere.dss.workflow.entity.request.UpdateExternalNodeRequest; +import com.webank.wedatasphere.dss.workflow.entity.vo.NodeGroupVO; +import com.webank.wedatasphere.dss.workflow.entity.vo.NodeInfoVO; +import com.webank.wedatasphere.dss.workflow.entity.vo.NodeUiVO; +import com.webank.wedatasphere.dss.workflow.entity.vo.NodeUiValidateVO; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import com.webank.wedatasphere.dss.workflow.service.WorkflowNodeService; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.ArrayUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@RestController +@RequestMapping(path = "/dss/workflow", produces = {"application/json"}) +public class NodeRestfulApi { + private Logger logger = LoggerFactory.getLogger(this.getClass()); + @Autowired + private DSSFlowService dssFlowService; + @Autowired + private WorkflowNodeService workflowNodeService; + @Autowired + private WorkFlowParser workFlowParser; + + @RequestMapping(value = "/listNodeType",method = RequestMethod.GET) + public Message listNodeType(HttpServletRequest req) { + Function supplier = internationalization(req, NodeGroup::getNameEn, NodeGroup::getName); + List groupVos = new ArrayList<>(); + //cache + List groups = workflowNodeService.listNodeGroups(); + for (NodeGroup group : groups) { + NodeGroupVO nodeGroupVO = new NodeGroupVO(); + BeanUtils.copyProperties(group, nodeGroupVO); + nodeGroupVO.setTitle(supplier.apply(group)); + nodeGroupVO.setChildren(group.getNodes().stream().map(n -> { + try { + return transfer(n, req); + } catch (IOException e) { + logger.error("listNodeType get icons failed.", e); + throw new DSSRuntimeException(81200, e.getMessage(), e); + } + }).collect(Collectors.toList())); + groupVos.add(nodeGroupVO); + } + groupVos = groupVos.stream().sorted(NodeGroupVO::compareTo).collect(Collectors.toList()); + return Message.ok().data("nodeTypes", groupVos); + } + + private Function internationalization(HttpServletRequest req, + Function supplier1, + Function supplier2) { + String language = req.getHeader("Content-language"); + if (language != null) { + language = language.trim(); + } + if (ContentLanguage.en.getName().equals(language)) { + return supplier1; + } else { + return supplier2; + } + } + + private NodeUiValidateVO transfer(NodeUiValidate v, HttpServletRequest req) { + Function supplier = internationalization(req, NodeUiValidate::getErrorMsgEn, + NodeUiValidate::getErrorMsg); + NodeUiValidateVO nodeUiValidateVO = new NodeUiValidateVO(); + BeanUtils.copyProperties(v, nodeUiValidateVO); + nodeUiValidateVO.setMessage(supplier.apply(v)); + return nodeUiValidateVO; + } + + private NodeInfoVO transfer(NodeInfo nodeInfo, HttpServletRequest req) throws IOException { + NodeInfoVO nodeInfoVO = new NodeInfoVO(); + BeanUtils.copyProperties(nodeInfo, nodeInfoVO); + nodeInfoVO.setTitle(nodeInfo.getName()); + nodeInfoVO.setType(nodeInfo.getNodeType()); + nodeInfoVO.setImage(getIcon(nodeInfo)); + Function descriptionSupplier = internationalization(req, NodeUi::getDescriptionEn, NodeUi::getDescription); + Function labelNameSupplier = internationalization(req, NodeUi::getLableNameEn, NodeUi::getLableName); + ArrayList nodeUiVOS = new ArrayList<>(); + for (NodeUi nodeUi : nodeInfo.getNodeUis()) { + NodeUiVO nodeUiVO = new NodeUiVO(); + BeanUtils.copyProperties(nodeUi, nodeUiVO); + nodeUiVO.setDesc(descriptionSupplier.apply(nodeUi)); + nodeUiVO.setLableName(labelNameSupplier.apply(nodeUi)); + nodeUiVO.setNodeUiValidateVOS(nodeUi.getNodeUiValidates().stream().map(v -> transfer(v, req)).sorted(NodeUiValidateVO::compareTo).collect(Collectors.toList())); + nodeUiVOS.add(nodeUiVO); + } + nodeUiVOS.sort(NodeUiVO::compareTo); + nodeInfoVO.setNodeUiVOS(nodeUiVOS); + return nodeInfoVO; + } + + private String getIcon(NodeInfo nodeInfo) throws IOException { + String appConnHomePath = AppConnManager.getAppConnManager().getAppConnHomePath(nodeInfo.getAppConnName()); + File iconPath = new File(appConnHomePath, nodeInfo.getIconPath()); + if(!iconPath.exists()) { + throw new IOException("Get icon failed. Caused by: " + iconPath + " not exists."); + } else if(!iconPath.isFile()) { + throw new IOException("Get icon failed. Caused by: " + iconPath + " is not a file."); + } + return FileUtils.readFileToString(iconPath); + } + + @RequestMapping(value = "/createAppConnNode",method = RequestMethod.POST) + public Message createExternalNode(HttpServletRequest req, @RequestBody CreateExternalNodeRequest createExternalNodeRequest) throws DSSErrorException, IllegalAccessException, ExternalOperationFailedException, InstantiationException { + String userName = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + Long projectId = createExternalNodeRequest.getProjectID(); + String nodeType = createExternalNodeRequest.getNodeType(); + Long flowId = createExternalNodeRequest.getFlowID(); + Map params = createExternalNodeRequest.getParams(); + String nodeId = createExternalNodeRequest.getNodeID(); + + logger.info("User {} try to create a {} node for workflow {} in projectId {}, params is {}.", + userName, nodeType, flowId, projectId, params); + CommonAppConnNode node = new CommonAppConnNode(); + node.setNodeType(nodeType); + node.setFlowId(flowId); + node.setProjectId(projectId); + //update by peaceWong add nodeId to appConnNode + node.setId(nodeId); + //update by shanhuang json中解析获取contextID,然后放到请求参数中 + DSSFlow dssFlow = dssFlowService.getFlow(flowId); + String flowContent = dssFlow.getFlowJson(); + String ContextIDStr = DSSCSHelper.getContextIDStrByJson(flowContent); + node.setContextId(ContextIDStr); + //补个flowName的信息..如果这里没查询就直接让前台传了 + node.setFlowName(dssFlow.getName()); + String label = createExternalNodeRequest.getLabels().getRoute(); + node.setDssLabels(Collections.singletonList(new EnvDSSLabel(label))); + node.setWorkspace(workspace); + node.setParams(params); + node.setJobContent(params); + //补充json信息,方便appConn去解析获取响应的值 + if(params.containsKey(DSSJobContentConstant.UP_STREAM_KEY)) { + List dssNodes = null; + if(params.get(DSSJobContentConstant.UP_STREAM_KEY).equals("empty")) { +// dssNodes = workFlowParser.getWorkFlowNodes(flowContent); + params.remove(DSSJobContentConstant.UP_STREAM_KEY); + logger.info("Create a node that is not bound to an upstream node."); + } else { + String[] upStreams = ((String) params.get(DSSJobContentConstant.UP_STREAM_KEY)).split(","); + dssNodes = workFlowParser.getWorkFlowNodes(flowContent).stream() + .filter(dssNode -> ArrayUtils.contains(upStreams, dssNode.getId())).collect(Collectors.toList()); + if(dssNodes.isEmpty() && upStreams.length > 0) { + return Message.error("create node failed! Caused by: the banding up-stream nodes are not exists(绑定的上游节点不存在)."); + } + params.put(DSSJobContentConstant.UP_STREAM_KEY, dssNodes); + } + + } + Map jobContent = workflowNodeService.createNode(userName, node); + return Message.ok().data("result", jobContent); + } + + @RequestMapping(value = "/updateAppConnNode",method = RequestMethod.POST) + public Message updateExternalNode(HttpServletRequest req, @RequestBody UpdateExternalNodeRequest updateExternalNodeRequest) throws IllegalAccessException, ExternalOperationFailedException, InstantiationException { + String userName = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + Long projectId = updateExternalNodeRequest.getProjectID(); + Long flowId = updateExternalNodeRequest.getFlowID(); + String nodeType = updateExternalNodeRequest.getNodeType(); + Map params = updateExternalNodeRequest.getParams(); + logger.info("User {} try to update ExternalNode request with projectId {}, flowId {}, params is {}, nodeType: {}.", + userName, projectId, flowId, params, nodeType); + CommonAppConnNode node = new CommonAppConnNode(); + node.setProjectId(projectId); + node.setNodeType(nodeType); + node.setFlowId(flowId); + String label = updateExternalNodeRequest.getLabels().getRoute(); + node.setDssLabels(Collections.singletonList(new EnvDSSLabel(label))); + node.setWorkspace(workspace); + node.setParams(params); + node.setJobContent(params); + workflowNodeService.updateNode(userName, node); + return Message.ok().data("result", node.getJobContent()); + } + + @RequestMapping(value = "/deleteAppConnNode",method = RequestMethod.POST) + public Message deleteExternalNode(HttpServletRequest req, @RequestBody UpdateExternalNodeRequest updateExternalNodeRequest) throws IllegalAccessException, ExternalOperationFailedException, InstantiationException { + String userName = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + Long projectId = updateExternalNodeRequest.getProjectID(); + Long flowId = updateExternalNodeRequest.getFlowID(); + String nodeType = updateExternalNodeRequest.getNodeType(); + Map params = updateExternalNodeRequest.getParams(); + logger.info("User {} with the workflow {} in project {} try to delete externalNode with params: {}, nodeType: {}.", + userName, flowId, projectId, params, nodeType); + CommonAppConnNode node = new CommonAppConnNode(); + String label = updateExternalNodeRequest.getLabels().getRoute(); + node.setDssLabels(Collections.singletonList(new EnvDSSLabel(label))); + node.setWorkspace(workspace); + node.setProjectId(projectId); + node.setNodeType(nodeType); + node.setJobContent(params); + node.setFlowId(flowId); + node.setName(updateExternalNodeRequest.getName()); + workflowNodeService.deleteNode(userName, node); + return Message.ok().data("result", node.getJobContent()); + } + + @RequestMapping(value = "/batchDeleteAppConnNode",method = RequestMethod.POST) + public Message batchDeleteAppConnNode(HttpServletRequest req, @RequestBody BatchDeleteAppConnNodeRequest batchDeleteAppConnNodeRequest) throws IllegalAccessException, ExternalOperationFailedException, InstantiationException { + String userName = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + List> jsonList = batchDeleteAppConnNodeRequest.getNodes(); + jsonList.forEach(json ->{ + Long projectId = Long.parseLong(json.get("projectID").toString()); + String nodeType = json.get("nodeType").toString(); + Long flowId = (Long) json.get("flowID"); + Map params = (Map) json.get("params"); + logger.info("User {} try to delete ExternalNode with projectId {}, flowId {}, params is {}, nodeType: {}.", + userName, projectId, flowId, params, nodeType); + CommonAppConnNode node = new CommonAppConnNode(); + String label = ((Map) json.get("labels")).get("route").toString(); + node.setDssLabels(Collections.singletonList(new EnvDSSLabel(label))); + node.setWorkspace(workspace); + node.setProjectId(projectId); + node.setNodeType(nodeType); + node.setJobContent(params); + node.setFlowId(flowId); + workflowNodeService.deleteNode(userName, node); + }); + + return Message.ok("success").data("result","success"); + } + + @RequestMapping(value = "/getAppConnNodeUrl",method = RequestMethod.POST) + public Message getAppConnNodeUrl(HttpServletRequest req, @RequestBody AppConnNodeUrlRequest appConnNodeUrlRequest) throws IllegalAccessException, ExternalOperationFailedException, InstantiationException { + String userName = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + Long projectId = appConnNodeUrlRequest.getProjectID(); + Long flowId = appConnNodeUrlRequest.getFlowId(); + String nodeType = appConnNodeUrlRequest.getNodeType(); + Map params = appConnNodeUrlRequest.getParams(); + logger.info("User {} with the workflow {} in project {} try to getAppConnNodeUrl with params: {}, nodeType: {}.", + userName, flowId, projectId, params, nodeType); + CommonAppConnNode node = new CommonAppConnNode(); + node.setWorkspace(workspace); + node.setProjectId(projectId); + node.setNodeType(nodeType); + node.setJobContent(params); + node.setFlowId(flowId); + String label = appConnNodeUrlRequest.getLabels().getRoute(); + node.setDssLabels(Collections.singletonList(new EnvDSSLabel(label))); + String jumpUrl = workflowNodeService.getNodeJumpUrl(params, node, userName); + return Message.ok().data("jumpUrl", jumpUrl); + } +} + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJob.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJob.java new file mode 100644 index 000000000..f34c8b116 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJob.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DssJob implements Runnable { + + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + protected String user; + protected Long id; + protected DssJobListener dssJobListener; + protected DssJobStatus status; + + public DssJobListener getDssJobListener() { + return dssJobListener; + } + + public void setDssJobListener(DssJobListener dssJobListener) { + this.dssJobListener = dssJobListener; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public DssJobStatus getStatus() { + return status; + } + + public void setStatus(DssJobStatus status) { + this.status = status; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobDeamon.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobDeamon.java new file mode 100644 index 000000000..16dead9b9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobDeamon.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + +import java.util.concurrent.Future; + +public class DssJobDeamon extends DssJob { + private Future future; + + private int time; + + public DssJobDeamon(Future future, int time) { + this.future = future; + this.time = time; + } + + @Override + public void run() { + logger.info("id:{}的job守护线程启动", id); + try { + Thread.sleep(1000L * time); + } catch (InterruptedException e) { + logger.error("deamon被打断了", e); + Thread.currentThread().interrupt(); + } + logger.info("开始取消id:{}的job线程", id); + future.cancel(true); + logger.info("成功取消id:{}的job线程", id); + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobHook.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobHook.java new file mode 100644 index 000000000..7716a67a2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobHook.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + +public interface DssJobHook { + void preExecute(DssJob job); + + void postExecute(DssJob job); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobInfo.java new file mode 100644 index 000000000..2df13ec92 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobInfo.java @@ -0,0 +1,79 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + + +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; + +import java.util.Date; + +public class DssJobInfo { + + private Long id; + private DssJobStatus status; + private Date createTime; + private Date updateTime; + private String msg; + + public Boolean timeout() { + return System.currentTimeMillis() - updateTime.getTime() > (int) DSSWorkFlowConstant.CACHE_TIMEOUT.getValue(); + } + + public Boolean isExecute() { + return status != null && (status == DssJobStatus.Inited || status == DssJobStatus.Running); + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public DssJobStatus getStatus() { + return status; + } + + public void setStatus(DssJobStatus status) { + this.status = status; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobListener.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobListener.java new file mode 100644 index 000000000..6d566ebe3 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobListener.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + +public interface DssJobListener { + + void onJobSucceed(DssJob job, String infoMsg); + + void onJobFailed(DssJob job, String errorMsg); + + void onJobRunning(DssJob job, String infoMsg); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobManager.java new file mode 100644 index 000000000..c4b0033fe --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobManager.java @@ -0,0 +1,172 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + + +public abstract class DssJobManager implements DssJobListener { + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final Map cacheMap = new ConcurrentHashMap<>(); + + private final Map secondCacheMap = new ConcurrentHashMap<>();//执行完成后将放入二级缓存,只有qurey的时候才去掉 + + private List jobHooks = new ArrayList<>(); + + public void addDssJobHook(DssJobHook jobHook) { + jobHooks.add(jobHook); + } + + { + logger.info("定时线程开启..."); + Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + synchronized (secondCacheMap) { + secondCacheMap.entrySet().stream() + .filter(f -> (f.getValue().getStatus() == DssJobStatus.Succeed || f.getValue().getStatus() == DssJobStatus.Failed) && f.getValue().timeout()) + .forEach(f -> { + logger.info("开始remove second cache过期记录:{},更新时间{}", f.getKey(), f.getValue().getUpdateTime()); + secondCacheMap.remove(f.getKey()); + }); + } + } + }, 1, 1, TimeUnit.MINUTES); + Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + synchronized (cacheMap) { + cacheMap.forEach((k, v) -> { + logger.info("cache中id为:{}状态为:{}更新时间为:{}", k, v.getStatus(), v.getUpdateTime()); + }); + } + } + }, 1, 10, TimeUnit.MINUTES); + } + + @Override + public void onJobSucceed(DssJob job, String infoMsg) { + // TODO: 2019/6/21 目前不用加锁 + job.setStatus(DssJobStatus.Succeed); + logger.info(infoMsg); + Date updateTime = new Date(); + DssJobInfo cache = cacheMap.get(job.getId()); + cache.setUpdateTime(updateTime); + cache.setStatus(DssJobStatus.Succeed); + synchronized (String.valueOf(job.getId()).intern()) { + secondCacheMap.put(job.getId(), cacheMap.remove(job.getId())); + } + jobHooks.forEach(h -> h.postExecute(job)); + } + + @Override + public void onJobFailed(DssJob job, String errorMsg) { + job.setStatus(DssJobStatus.Failed); + logger.info(errorMsg); + Date updateTime = new Date(); + DssJobInfo cache = cacheMap.get(job.getId()); + cache.setUpdateTime(updateTime); + cache.setStatus(DssJobStatus.Failed); + cache.setMsg(errorMsg); + synchronized (String.valueOf(job.getId()).intern()) { + secondCacheMap.put(job.getId(), cacheMap.remove(job.getId())); + } + jobHooks.forEach(h -> h.postExecute(job)); + } + + @Override + public void onJobRunning(DssJob job, String infoMsg) { + job.setStatus(DssJobStatus.Running); + logger.info(infoMsg); + Date updateTime = new Date(); + DssJobInfo cache = cacheMap.get(job.getId()); + cache.setUpdateTime(updateTime); + cache.setStatus(DssJobStatus.Running); + } + + protected void addPublishCache(DssJobInfo job) throws DSSErrorException { + job.setStatus(DssJobStatus.Inited); + job.setCreateTime(new Date()); + job.setUpdateTime(new Date()); + cacheMap.put(job.getId(), job); + } + + private DssJobInfo getJobInfoCache(Long id) { + return cacheMap.get(id); + } + + private DssJobInfo getSecondJobInfoCache(Long id) { + return secondCacheMap.get(id); + } + + public void checkeIsExecuting(Long id) throws DSSErrorException { + DssJobInfo jobInfo = getJobInfoCache(id); + DssJobInfo seCondInfo = getSecondJobInfoCache(id); + if (jobInfo != null && jobInfo.isExecute() || seCondInfo != null) { + DssJobStatus status = jobInfo == null ? seCondInfo.getStatus() : jobInfo.getStatus(); + logger.info("job执行中,id:{},状态为{}", id, status); + throw new DSSErrorException(90014, "job执行中,不允许执行当前操作"); + } + } + + private DssJobInfo createJobInfo(Long id) { + DssJobInfo jobInfo = new DssJobInfo(); + jobInfo.setId(id); + return jobInfo; + } + + public void submit(DssJob job) throws DSSErrorException { + synchronized (String.valueOf(job.getId()).intern()) { + checkeIsExecuting(job.getId()); + addPublishCache(createJobInfo(job.getId())); + } + job.setStatus(DssJobStatus.Inited); + jobHooks.forEach(h -> h.preExecute(job)); + Future submit = DssJobThreadPool.get().submit(job); + DssJobDeamon deamon = new DssJobDeamon(submit, (int) DSSWorkFlowConstant.PUBLISH_TIMEOUT.getValue()); + deamon.setId(job.getId()); + DssJobThreadPool.getDeamon().execute(deamon); + } + + public DssJobInfo getJobStatus(Long id) throws DSSErrorException { + synchronized (String.valueOf(id).intern()) { + DssJobInfo jobInfo = getJobInfoCache(id); + if (jobInfo != null) { + return jobInfo; + } else { + DssJobInfo remove = secondCacheMap.remove(id); + if (remove == null) throw new DSSErrorException(90021, String.format("the id %s is not exist", id)); + return remove; + } + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobStatus.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobStatus.java new file mode 100644 index 000000000..92209e23b --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobStatus.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + +public enum DssJobStatus { + Inited,Running,Succeed,Failed +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobThreadPool.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobThreadPool.java new file mode 100644 index 000000000..e71556661 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobThreadPool.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + +import org.apache.linkis.common.utils.Utils; + +import java.util.concurrent.ExecutorService; + +import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.NODE_EXPORT_IMPORT_THREAD_NUM; + +public class DssJobThreadPool { + + private static ExecutorService executorService = Utils.newFixedThreadPool(1000, "project-publish", false); + private static ExecutorService executorServiceDeamon = Utils.newFixedThreadPool(1000, "project-publish-deamon", true); + public static ExecutorService nodeExportThreadPool = Utils.newFixedThreadPool(NODE_EXPORT_IMPORT_THREAD_NUM.getValue(), "workflowNode-ExportImport-Thread-", false); + + public static ExecutorService get() { + return executorService; + } + + public static ExecutorService getDeamon() { + return executorServiceDeamon; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/JobParams.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/JobParams.java new file mode 100644 index 000000000..4f9af649c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/JobParams.java @@ -0,0 +1,20 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.scheduler; + +public interface JobParams { +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/DSSFlowService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/DSSFlowService.java new file mode 100644 index 000000000..1907728d7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/DSSFlowService.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.entity.vo.ExtraToolBarsVO; + +import java.io.IOException; +import java.util.List; + +public interface DSSFlowService { + DSSFlow getFlowByID(Long id); + + DSSFlow getFlowWithJsonAndSubFlowsByID(Long rootFlowId); + + DSSFlow addFlow(DSSFlow dssFlow, String contextIdStr, String orcVersion, String schedulerAppConn) throws DSSErrorException; + + DSSFlow addSubFlow(DSSFlow dssFlow, Long parentFlowId, String contextIdStr, String orcVersion, String schedulerAppConn) throws DSSErrorException; + + /** + * 通过flowID获取最新版本的dwsFlow,版本信息在latestVersion + * @return + */ + DSSFlow getFlow(Long flowId); + + void updateFlowBaseInfo(DSSFlow dssFlow) throws DSSErrorException; + + + void batchDeleteFlow(List flowIdlist); + + String saveFlow(Long flowId, + String jsonFlow, + String comment, + String userName, + String workspaceName, + String projectName + ) throws DSSErrorException; + + DSSFlow copyRootFlow(Long rootFlowId, String userName, Workspace workspace, + String projectName, String version, String contextIdStr, + String description, List dssLabels) throws DSSErrorException, IOException; + + Long getParentFlowID(Long id); + + List getExtraToolBars(long workspaceId, long projectId); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/PublishService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/PublishService.java new file mode 100644 index 000000000..a458c7098 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/PublishService.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.service; + +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.Map; + +public interface PublishService { + + + String submitPublish(String publishUser, Long workflowId, + Map dssLabel, Workspace workspace, String comment) throws Exception; + + ResponseConvertOrchestrator getStatus(String username, String taskId) throws Exception; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/WorkflowNodeService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/WorkflowNodeService.java new file mode 100644 index 000000000..89e87f00e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/WorkflowNodeService.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.standard.app.development.ref.ExportResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.workflow.entity.AbstractAppConnNode; +import com.webank.wedatasphere.dss.workflow.entity.CommonAppConnNode; +import com.webank.wedatasphere.dss.workflow.entity.NodeGroup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +public interface WorkflowNodeService { + + List listNodeGroups(); + + /** + * 根据参数创建外部节点 + * @param node 节点信息 + * @return 返回jobContent的Map,工作流会将该Map存储起来,作为该节点的关键关联信息,用于后续的CRUD和执行。 + */ + Map createNode(String userName, CommonAppConnNode node) throws ExternalOperationFailedException; + + void deleteNode(String userName, CommonAppConnNode node) throws ExternalOperationFailedException; + + Map updateNode(String userName, CommonAppConnNode node) throws ExternalOperationFailedException; + + default Map refresh(String userName, CommonAppConnNode node) { + return null; + } + + Map copyNode(String userName, CommonAppConnNode newNode, CommonAppConnNode oldNode, String orcVersion) throws IOException, DSSErrorException; + + default void setNodeReadOnly(String userName, CommonAppConnNode node) { + } + + default List listNodes(String userName, CommonAppConnNode node) { + return new ArrayList<>(); + } + + Map importNode(String userName, + CommonAppConnNode node, + Supplier> getBmlResourceMap, + Supplier> getStreamResourceMap, + String orcVersion) throws Exception; + + ExportResponseRef exportNode(String userName, CommonAppConnNode node); + + String getNodeJumpUrl(Map params, CommonAppConnNode node, String userName) throws ExternalOperationFailedException; + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java new file mode 100644 index 000000000..16d4c7d8f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java @@ -0,0 +1,515 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.service.impl; + + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.entity.node.DSSNode; +import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import com.webank.wedatasphere.dss.standard.app.development.utils.DSSJobContentConstant; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.dao.NodeInfoMapper; +import com.webank.wedatasphere.dss.workflow.entity.CommonAppConnNode; +import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; +import com.webank.wedatasphere.dss.workflow.entity.vo.ExtraToolBarsVO; +import com.webank.wedatasphere.dss.workflow.io.export.NodeExportService; +import com.webank.wedatasphere.dss.workflow.io.input.NodeInputService; +import com.webank.wedatasphere.dss.workflow.lock.Lock; +import com.webank.wedatasphere.dss.workflow.service.BMLService; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import com.webank.wedatasphere.dss.workflow.service.WorkflowNodeService; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.cs.common.utils.CSCommonUtils; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.stream.Collectors; + +import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.SCHEDULER_APP_CONN_NAME; + +@Service +public class DSSFlowServiceImpl implements DSSFlowService { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + @Autowired + private FlowMapper flowMapper; + @Autowired + private NodeInfoMapper nodeInfoMapper; + @Autowired + private NodeInputService nodeInputService; + + @Autowired + private WorkFlowParser workFlowParser; + @Autowired + private BMLService bmlService; + @Autowired + private NodeExportService nodeExportService; + @Autowired + private WorkflowNodeService workflowNodeService; + + private static ContextService contextService = ContextServiceImpl.getInstance(); + + private static final String TOKEN = CommonVars.apply("wds.dss.workspace.token", "BML-AUTH").getValue(); + + @Override + public DSSFlow getFlowByID(Long id) { + return flowMapper.selectFlowByID(id); + } + + @Override + public DSSFlow getFlowWithJsonAndSubFlowsByID(Long rootFlowId) { + return genDSSFlowTree(rootFlowId); + } + + private DSSFlow genDSSFlowTree(Long parentFlowId) { + DSSFlow cyFlow = flowMapper.selectFlowByID(parentFlowId); + + String userName = cyFlow.getCreator(); + Map query = bmlService.query(userName, cyFlow.getResourceId(), cyFlow.getBmlVersion()); + cyFlow.setFlowJson(query.get("string").toString()); + + List subFlowIDs = flowMapper.selectSubFlowIDByParentFlowID(parentFlowId); + logger.info("find subFlow_ids:{} for parentFlow_id:{}", subFlowIDs, parentFlowId); + for (Long subFlowID : subFlowIDs) { + if (cyFlow.getChildren() == null) { + cyFlow.setChildren(new ArrayList()); + } + DSSFlow subFlow = genDSSFlowTree(subFlowID); + + cyFlow.addChildren(subFlow); + } + return cyFlow; + } + + @Lock + @Transactional(rollbackFor = DSSErrorException.class) + @Override + public DSSFlow addFlow(DSSFlow dssFlow, + String contextID, String orcVersion, + String schedulerAppConn) throws DSSErrorException { + try { + flowMapper.insertFlow(dssFlow); + } catch (DuplicateKeyException e) { + logger.info(e.getMessage()); + throw new DSSErrorException(90003, "工作流名不能重复"); + } + Map flowJsonMap = new HashMap<>(); + String userName = dssFlow.getCreator(); + if (StringUtils.isNotBlank(contextID)) { + flowJsonMap.put(CSCommonUtils.CONTEXT_ID_STR, contextID); + } + if (StringUtils.isNotBlank(orcVersion)) { + flowJsonMap.put(DSSJobContentConstant.ORC_VERSION_KEY, orcVersion); + } + if (StringUtils.isNotBlank(schedulerAppConn)) { + flowJsonMap.put(SCHEDULER_APP_CONN_NAME, schedulerAppConn); + } + String jsonFlow = DSSCommonUtils.COMMON_GSON.toJson(flowJsonMap); + Map bmlReturnMap = bmlService.upload(userName, jsonFlow, UUID.randomUUID().toString() + ".json", + dssFlow.getName()); + + dssFlow.setResourceId(bmlReturnMap.get("resourceId").toString()); + + dssFlow.setBmlVersion(bmlReturnMap.get("version").toString()); + Long parentFlowID = null; + if (!dssFlow.getRootFlow()) { + parentFlowID = flowMapper.getParentFlowID(dssFlow.getId()); + } + //todo update dssflow + contextService.checkAndSaveContext(jsonFlow, String.valueOf(parentFlowID)); + flowMapper.updateFlowInputInfo(dssFlow); + return dssFlow; + } + + @Lock + @Transactional(rollbackFor = DSSErrorException.class) + @Override + public DSSFlow addSubFlow(DSSFlow DSSFlow, Long parentFlowID, String contextIDStr, String orcVersion, String schedulerAppConn) throws DSSErrorException { + DSSFlow parentFlow = flowMapper.selectFlowByID(parentFlowID); + DSSFlow.setProjectID(parentFlow.getProjectID()); + DSSFlow subFlow = addFlow(DSSFlow, contextIDStr, orcVersion, schedulerAppConn); + //数据库中插入关联信息 + flowMapper.insertFlowRelation(subFlow.getId(), parentFlowID); + return subFlow; + } + + @Override + public DSSFlow getFlow(Long flowID) { + DSSFlow DSSFlow = getFlowByID(flowID); + //todo update + String userName = DSSFlow.getCreator(); + Map query = bmlService.query(userName, DSSFlow.getResourceId(), DSSFlow.getBmlVersion()); + DSSFlow.setFlowJson(query.get("string").toString()); + return DSSFlow; + } + + @Lock + @Transactional(rollbackFor = DSSErrorException.class) + @Override + public void updateFlowBaseInfo(DSSFlow DSSFlow) throws DSSErrorException { + try { + flowMapper.updateFlowBaseInfo(DSSFlow); + } catch (DuplicateKeyException e) { + logger.info(e.getMessage()); + throw new DSSErrorException(90003, "工作流名不能重复"); + } + } + + @Lock + @Transactional(rollbackFor = DSSErrorException.class) + @Override + public void batchDeleteFlow(List flowIDlist) { + flowIDlist.forEach(this::deleteFlow); + } + + /*@Lock*/ + @Transactional(rollbackFor = {DSSErrorException.class}) + @Override + public String saveFlow(Long flowID, + String jsonFlow, + String comment, + String userName, + String workspaceName, + String projectName) { + + DSSFlow dssFlow = flowMapper.selectFlowByID(flowID); + String creator = dssFlow.getCreator(); + if (StringUtils.isNotEmpty(creator)) { + userName = creator; + } + String flowJsonOld = getFlowJson(userName, projectName, dssFlow); + if (isEqualTwoJson(flowJsonOld, jsonFlow)) { + logger.info("saveFlow is not change"); + return dssFlow.getBmlVersion(); + } else { + logger.info("saveFlow is change"); + } + String resourceId = dssFlow.getResourceId(); + Long parentFlowID = flowMapper.getParentFlowID(flowID); + // 这里不要检查ContextID具体版本等,只要存在就不创建 2020-0423 + jsonFlow = contextService.checkAndCreateContextID(jsonFlow, dssFlow.getBmlVersion(), + workspaceName, projectName, dssFlow.getName(), userName, false); + + Map bmlReturnMap = bmlService.update(userName, resourceId, jsonFlow); + + dssFlow.setId(flowID); + dssFlow.setHasSaved(true); + dssFlow.setDescription(comment); + dssFlow.setResourceId(bmlReturnMap.get("resourceId").toString()); + dssFlow.setBmlVersion(bmlReturnMap.get("version").toString()); + //todo 数据库增加版本更新 + flowMapper.updateFlowInputInfo(dssFlow); + + try { + contextService.checkAndSaveContext(jsonFlow, String.valueOf(parentFlowID)); + } catch (DSSErrorException e) { + logger.error("Failed to saveContext: ", e); + } + + String version = bmlReturnMap.get("version").toString(); + return version; + } + + public boolean isEqualTwoJson(String oldJsonNode, String newJsonNode) { + Gson gson = new Gson(); + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(oldJsonNode).getAsJsonObject(); + jsonObject.remove("updateTime"); + jsonObject.remove("comment"); + String tempOldJson = gson.toJson(jsonObject); + + JsonObject jsonObject2 = parser.parse(newJsonNode).getAsJsonObject(); + jsonObject2.remove("updateTime"); + jsonObject2.remove("comment"); + String tempNewJson = gson.toJson(jsonObject2); + return tempOldJson.equals(tempNewJson); + } + + public String getFlowJson(String userName, String projectName, DSSFlow dssFlow) { + String flowExportSaveBasePath = IoUtils.generateIOPath(userName, projectName, ""); + String savePath = flowExportSaveBasePath + File.separator + dssFlow.getName() + File.separator + dssFlow.getName() + ".json"; + String flowJson = bmlService.downloadAndGetFlowJson(userName, dssFlow.getResourceId(), dssFlow.getBmlVersion(), savePath); + return flowJson; + } + + @Override + public Long getParentFlowID(Long flowID) { + return flowMapper.getParentFlowID(flowID); + } + + @Override + public List getExtraToolBars(long workspaceId, long projectId) { + List retList = new ArrayList<>(); + retList.add(new ExtraToolBarsVO("前往调度中心", DSSWorkFlowConstant.GOTO_SCHEDULER_CENTER_URL.getValue() + "?workspaceId=" + workspaceId, "icon:null")); + return retList; + } + + + public void deleteFlow(Long flowId) { + List subFlowIDs = flowMapper.selectSubFlowIDByParentFlowID(flowId); + for (Long subFlowID : subFlowIDs) { + deleteFlow(subFlowID); + } + for (Long subFlowID : subFlowIDs) { + deleteDWSDB(subFlowID); + // TODO: 2019/6/5 wtss发布过的工作流的删除? + // TODO: 2019/6/5 json中资源的删除 + // TODO: 2019/6/5 事务的保证 + } + deleteDWSDB(flowId); + } + + private void deleteDWSDB(Long flowID) { + flowMapper.deleteFlowBaseInfo(flowID); + flowMapper.deleteFlowRelation(flowID); + //第一期没有工作流的发布,所以不需要删除dws工作流的发布表 + } + + + @Override + public DSSFlow copyRootFlow(Long rootFlowId, String userName, Workspace workspace, + String projectName, String version, String contextIdStr, + String description, List dssLabels) throws DSSErrorException, IOException { + DSSFlow dssFlow = flowMapper.selectFlowByID(rootFlowId); + if(dssFlow == null) { + throw new DSSErrorException(50030, "Workflow " + rootFlowId + " is not exists[工作流不存在,请检查是否已被删除]."); + } + logger.info("User {} try to copy workflow {} in project {} with newVersion {} and contextId {}.", userName, + dssFlow.getName(), projectName, version, contextIdStr); + DSSFlow rootFlowWithSubFlows = copyFlowAndSetSubFlowInDB(dssFlow, userName, description); + updateFlowJson(userName, projectName, rootFlowWithSubFlows, version, null, + contextIdStr, workspace, dssLabels); + return flowMapper.selectFlowByID(rootFlowWithSubFlows.getId()); + } + + private DSSFlow copyFlowAndSetSubFlowInDB(DSSFlow dssFlow, + String userName, String description) { + DSSFlow cyFlow = new DSSFlow(); + BeanUtils.copyProperties(dssFlow, cyFlow, "children", "flowVersions"); + //封装flow信息 + cyFlow.setCreator(userName); + cyFlow.setCreateTime(new Date()); + if (StringUtils.isNotBlank(description)) { + cyFlow.setDescription(description); + } + cyFlow.setId(null); + flowMapper.insertFlow(cyFlow); + List subFlowIDs = flowMapper.selectSubFlowIDByParentFlowID(dssFlow.getId()); + for (Long subFlowID : subFlowIDs) { + DSSFlow subDSSFlow = flowMapper.selectFlowByID(subFlowID); + if (dssFlow.getChildren() == null) { + dssFlow.setChildren(new ArrayList()); + } + DSSFlow copySubFlow = copyFlowAndSetSubFlowInDB(subDSSFlow, userName, description); + persistenceFlowRelation(copySubFlow.getId(), cyFlow.getId()); + cyFlow.addChildren(copySubFlow); + } + return cyFlow; + } + + private void updateFlowJson(String userName, String projectName, DSSFlow rootFlow, + String version, Long parentFlowId, String contextIdStr, + Workspace workspace, List dssLabels) throws DSSErrorException, IOException { + String flowJson = bmlService.readFlowJsonFromBML(userName, rootFlow.getResourceId(), rootFlow.getBmlVersion()); + //如果包含subflow,需要一同导入subflow内容,并更新parrentflow的json内容 + // TODO: 2020/7/31 优化update方法里面的saveContent + String updateFlowJson = updateFlowContextIdAndVersion(flowJson, contextIdStr, version); + //重新上传工作流资源 + updateFlowJson = uploadFlowResourceToBml(userName, updateFlowJson, projectName, rootFlow); + //上传节点的资源或调用appconn的copyRef + updateFlowJson = updateWorkFlowNodeJson(userName, projectName, updateFlowJson, rootFlow, + version, workspace, dssLabels); + List subFlows = rootFlow.getChildren(); + if (subFlows != null) { + for (DSSFlow subflow : subFlows) { + updateFlowJson(userName, projectName, subflow, version, rootFlow.getId(), + contextIdStr, workspace, dssLabels); + } + } + + DSSFlow updateDssFlow = uploadFlowJsonToBml(userName, projectName, rootFlow, updateFlowJson); + //todo add dssflow to database + flowMapper.updateFlowInputInfo(updateDssFlow); + contextService.checkAndSaveContext(updateFlowJson, String.valueOf(parentFlowId)); + } + + //上传工作流资源 + public String uploadFlowResourceToBml(String userName, String flowJson, String projectName, DSSFlow dssFlow) throws IOException { + List resourceList = workFlowParser.getWorkFlowResources(flowJson); + if (CollectionUtils.isEmpty(resourceList)) { + return flowJson; + } + String flowExportSaveBasePath = IoUtils.generateIOPath(userName, projectName, ""); + String savePath = flowExportSaveBasePath + File.separator + dssFlow.getName() + File.separator + "resource"; + //上传文件获取resourceId和version save应该是已经有 + resourceList.forEach(resource -> { + //从bml取出resource + String flowResourcePath = savePath + File.separator + resource.getResourceId() + ".re"; + bmlService.downloadToLocalPath(userName, resource.getResourceId(), resource.getVersion(), flowResourcePath); + //重新上传resource + InputStream resourceInputStream = bmlService.readLocalResourceFile(userName, flowResourcePath); + Map bmlReturnMap = bmlService.upload(userName, resourceInputStream, UUID.randomUUID().toString() + ".json", projectName); + resource.setResourceId(bmlReturnMap.get("resourceId").toString()); + resource.setVersion(bmlReturnMap.get("version").toString()); + }); + //更新flowJson的resources + return workFlowParser.updateFlowJsonWithKey(flowJson, "resources", resourceList); + } + + + private DSSFlow uploadFlowJsonToBml(String userName, String projectName, DSSFlow dssFlow, String flowJson) throws IOException { + //上传文件获取resourceId和version save应该是已经有 + Map bmlReturnMap; + + //上传文件获取resourceId和version save应该是已经有 + bmlReturnMap = bmlService.upload(userName, flowJson, UUID.randomUUID().toString() + ".json", projectName); + + dssFlow.setCreator(userName); + dssFlow.setBmlVersion(bmlReturnMap.get("version").toString()); + dssFlow.setResourceId(bmlReturnMap.get("resourceId").toString()); + dssFlow.setDescription("copy update workflow"); + dssFlow.setSource("copy更新"); + //version表中插入数据 + return dssFlow; + } + + private String updateFlowContextIdAndVersion(String flowJson, String contextIdStr, String orcVersion) throws IOException { + return workFlowParser.updateFlowJsonWithMap(flowJson, MapUtils.newCommonMap(CSCommonUtils.CONTEXT_ID_STR, contextIdStr, + DSSJobContentConstant.ORC_VERSION_KEY, orcVersion)); + } + + private String updateWorkFlowNodeJson(String userName, String projectName, + String flowJson, DSSFlow dssFlow, + String version, Workspace workspace, List dssLabels) throws DSSErrorException, IOException { + if (StringUtils.isEmpty(version)) { + logger.warn("version id is null when updateWorkFlowNodeJson"); + version = String.valueOf((new Random().nextLong())); + } + List nodeJsonList = workFlowParser.getWorkFlowNodesJson(flowJson); + if (nodeJsonList == null) { + throw new DSSErrorException(90073, "工作流内没有工作流节点,导入失败." + dssFlow.getName()); + } + String updateContextId = workFlowParser.getValueWithKey(flowJson, CSCommonUtils.CONTEXT_ID_STR); + if (nodeJsonList.size() == 0) { + return flowJson; + } + List subflows = (List) dssFlow.getChildren(); + List> nodeJsonListRes = new ArrayList<>(); + + for (String nodeJson : nodeJsonList) { + //重新上传一份jar文件到bml + String updateNodeJson = inputNodeFiles(userName, projectName, nodeJson); + + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + //更新subflowID + String nodeType = nodeJsonMap.get("jobType").toString(); + NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(nodeType); + if ("workflow.subflow".equals(nodeType)) { + String subFlowName = nodeJsonMap.get("title").toString(); + List dssFlowList = subflows.stream().filter(subflow -> + subflow.getName().equals(subFlowName) + ).collect(Collectors.toList()); + if (dssFlowList.size() == 1) { + updateNodeJson = nodeInputService.updateNodeSubflowID(updateNodeJson, dssFlowList.get(0).getId()); + nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + nodeJsonListRes.add(nodeJsonMap); + } else if (dssFlowList.size() > 1) { + logger.error("工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + throw new DSSErrorException(90077, "工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + } else { + logger.error("工程内存在重复的子工作流节点名称,导入失败" + subFlowName); + throw new DSSErrorException(90078, "工程内未能找到子工作流节点,导入失败" + subFlowName); + } +// } else if (nodeJsonMap.get("jobContent") != null && !((Map) nodeJsonMap.get("jobContent")).containsKey("script")) { + } else if (Boolean.TRUE.equals(nodeInfo.getSupportJump()) && nodeInfo.getJumpType() == 1) { + logger.info("nodeJsonMap.jobContent is:{}", nodeJsonMap.get("jobContent")); + CommonAppConnNode newNode = new CommonAppConnNode(); + CommonAppConnNode oldNode = new CommonAppConnNode(); + oldNode.setJobContent((Map) nodeJsonMap.get("jobContent")); + oldNode.setContextId(updateContextId); + oldNode.setNodeType(nodeType); + oldNode.setName((String) nodeJsonMap.get("title")); + oldNode.setFlowId(dssFlow.getId()); + oldNode.setWorkspace(workspace); + oldNode.setDssLabels(dssLabels); + oldNode.setFlowName(dssFlow.getName()); + oldNode.setProjectId(dssFlow.getProjectID()); + oldNode.setProjectName(projectName); + newNode.setName(oldNode.getName()); + Map jobContent = workflowNodeService.copyNode(userName, newNode, oldNode, version); + nodeJsonMap.put("jobContent", jobContent); + nodeJsonListRes.add(nodeJsonMap); + } else { + nodeJsonListRes.add(nodeJsonMap); + } + } + + return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); + } + + //由于每一个节点可能含有jar文件,这个功能不能直接复制使用,因为删掉新版本节点会直接删掉旧版本的node中的jar文件 + //所以重新上传一份jar文件到bml + private String inputNodeFiles(String userName, String projectName, String nodeJson) throws IOException { + String flowPath = IoUtils.generateIOPath(userName, projectName, ""); + String workFlowResourceSavePath = flowPath + File.separator + "resource" + File.separator; + Gson gson = new Gson(); + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(nodeJson).getAsJsonObject(); + DSSNode node = gson.fromJson(jsonObject, new TypeToken() { + }.getType()); + //先导出来 + nodeExportService.downloadNodeResourceToLocal(userName, node, workFlowResourceSavePath); + //后导入到bml + String updateNodeJson = nodeInputService.uploadResourceToBml(userName, nodeJson, workFlowResourceSavePath, projectName); + return updateNodeJson; + } + + private void persistenceFlowRelation(Long flowID, Long parentFlowID) { + DSSFlowRelation relation = flowMapper.selectFlowRelation(flowID, parentFlowID); + if (relation == null) { + flowMapper.insertFlowRelation(flowID, parentFlowID); + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java new file mode 100644 index 000000000..1600aad41 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java @@ -0,0 +1,126 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.service.impl; + +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestrationStatus; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import com.webank.wedatasphere.dss.workflow.service.PublishService; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.rpc.Sender; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Map; + +import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.DEFAULT_SCHEDULER_APP_CONN; + +public class PublishServiceImpl implements PublishService { + + @Autowired + private DSSFlowService dssFlowService; + @Autowired + private WorkFlowParser workFlowParser; + + public void setDssFlowService(DSSFlowService dssFlowService) { + this.dssFlowService = dssFlowService; + } + + public void setWorkFlowParser(WorkFlowParser workFlowParser) { + this.workFlowParser = workFlowParser; + } + + protected final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + protected Sender getOrchestratorSender() { + return DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + } + + @Override + public String submitPublish(String convertUser, Long workflowId, + Map dssLabel, Workspace workspace, String comment) throws Exception { + LOGGER.info("User {} begins to convert workflow {}.", convertUser, workflowId); + //1 获取对应的orcId 和 orcVersionId + //2.进行提交 + RequestFrameworkConvertOrchestration requestFrameworkConvertOrchestration = new RequestFrameworkConvertOrchestration(); + requestFrameworkConvertOrchestration.setComment(comment); + requestFrameworkConvertOrchestration.setOrcAppId(workflowId); + requestFrameworkConvertOrchestration.setUserName(convertUser); + requestFrameworkConvertOrchestration.setWorkspace(workspace); + DSSFlow dssFlow = null; + try { + dssFlow = dssFlowService.getFlow(workflowId); + if(dssFlow == null) { + DSSExceptionUtils.dealErrorException(63325, "workflow " + workflowId + " is not exists.", DSSErrorException.class); + return null; + } + String schedulerAppConnName = workFlowParser.getValueWithKey(dssFlow.getFlowJson(), DSSWorkFlowConstant.SCHEDULER_APP_CONN_NAME); + if(StringUtils.isBlank(schedulerAppConnName)) { + // 向下兼容老版本 + schedulerAppConnName = DEFAULT_SCHEDULER_APP_CONN.getValue(); + } + SchedulerAppConn schedulerAppConn = (SchedulerAppConn) AppConnManager.getAppConnManager().getAppConn(schedulerAppConnName); + // 只是为了获取是否需要发布所有Orc,这里直接拿第一个AppInstance即可。 + AppInstance appInstance = schedulerAppConn.getAppDesc().getAppInstances().get(0); + requestFrameworkConvertOrchestration.setConvertAllOrcs(schedulerAppConn.getOrCreateConversionStandard().getDSSToRelConversionService(appInstance).isConvertAllOrcs()); + requestFrameworkConvertOrchestration.setLabels(dssLabel); + ResponseConvertOrchestrator response = (ResponseConvertOrchestrator) getOrchestratorSender().ask(requestFrameworkConvertOrchestration); + if(response.getResponse().isFailed()) { + throw new DSSErrorException(50311, response.getResponse().getMessage()); + } + return response.getId(); + } catch (DSSErrorException e) { + throw e; + } catch (final Exception t) { + Object str = dssFlow == null ? workflowId : dssFlow.getName(); + LOGGER.error("User {} failed to submit publish {}.", convertUser, str, t); + DSSExceptionUtils.dealErrorException(63325, "Failed to submit publish " + str, t, DSSErrorException.class); + } + return null; + } + + @Override + public ResponseConvertOrchestrator getStatus(String username, String taskId) { + if (LOGGER.isDebugEnabled()){ + LOGGER.debug("{} is asking status of {}.", username, taskId); + } + ResponseConvertOrchestrator response =new ResponseConvertOrchestrator(); + //通过rpc的方式去获取到最新status + try { + RequestFrameworkConvertOrchestrationStatus req = new RequestFrameworkConvertOrchestrationStatus(taskId); + response = (ResponseConvertOrchestrator) getOrchestratorSender().ask(req); + LOGGER.info("user {} gets status of {}, status is {},msg is {}", username, taskId, response.getResponse().getJobStatus(), response.getResponse().getMessage()); + }catch (Exception t){ + LOGGER.error("failed to getStatus {} ", taskId, t); + + } + return response; + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/WorkflowNodeServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/WorkflowNodeServiceImpl.java new file mode 100644 index 000000000..602ea350a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/WorkflowNodeServiceImpl.java @@ -0,0 +1,295 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.service.impl; + +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlySSOAppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectRelationRequest; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectRelationResponse; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.*; +import com.webank.wedatasphere.dss.standard.app.development.ref.*; +import com.webank.wedatasphere.dss.standard.app.development.service.*; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.development.utils.DSSJobContentConstant; +import com.webank.wedatasphere.dss.standard.app.development.utils.DevelopmentOperationUtils; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.NoSuchAppInstanceException; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.dao.NodeInfoMapper; +import com.webank.wedatasphere.dss.workflow.entity.AbstractAppConnNode; +import com.webank.wedatasphere.dss.workflow.entity.CommonAppConnNode; +import com.webank.wedatasphere.dss.workflow.entity.NodeGroup; +import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import com.webank.wedatasphere.dss.workflow.service.WorkflowNodeService; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.rpc.Sender; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; + +@Service +public class WorkflowNodeServiceImpl implements WorkflowNodeService { + + @Autowired + private NodeInfoMapper nodeInfoMapper; + @Autowired + private FlowMapper flowMapper; + @Autowired + private WorkFlowParser workFlowParser; + @Autowired + private DSSFlowService dssFlowService; + + private Sender projectSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender(); + + @Override + public List listNodeGroups() { + //cache + return nodeInfoMapper.listNodeGroups(); + } + + private RefCRUDService getRefCRUDService(AppConn appConn, List dssLabels) { + return getDevelopmentService(appConn, dssLabels, DevelopmentIntegrationStandard::getRefCRUDService); + } + + private T getDevelopmentService(AppConn appConn, List dssLabels, + BiFunction getDevelopmentService) { + DevelopmentIntegrationStandard developmentIntegrationStandard = ((OnlyDevelopmentAppConn) appConn).getOrCreateDevelopmentStandard(); + if (developmentIntegrationStandard == null) { + throw new ExternalOperationFailedException(50020, appConn.getAppDesc().getAppName() + " does not exists development standard, please ask admin to check the AppConn."); + } + AppInstance appInstance; + try { + appInstance = appConn.getAppDesc().getAppInstancesByLabels(dssLabels).get(0); + } catch (NoSuchAppInstanceException e) { + throw new ExternalOperationFailedException(50020, "cannot find the appInstance with label " + dssLabels.get(0).getStringValue()); + } + return getDevelopmentService.apply(developmentIntegrationStandard, appInstance); + } + + @Override + public Map createNode(String userName, CommonAppConnNode node) throws ExternalOperationFailedException { + RefJobContentResponseRef responseRef = tryNodeOperation(userName, node, this::getRefCRUDService, + developmentService -> ((RefCRUDService) developmentService).getRefCreationOperation(), + (developmentOperation, developmentRequestRef) -> + ((RefCreationOperation) developmentOperation).createRef((DSSJobContentRequestRef) developmentRequestRef) + , (developmentRequestRef, refJobContentResponseRef) -> { + if (developmentRequestRef instanceof ProjectRefRequestRef) { + Long projectRefId = ((ProjectRefRequestRef) developmentRequestRef).getRefProjectId(); + refJobContentResponseRef.getRefJobContent().put(DSSWorkFlowConstant.REF_PROJECT_ID_KEY, projectRefId); + } + }, "create"); + return responseRef.getRefJobContent(); + } + + private V tryNodeOperation(String userName, CommonAppConnNode node, + BiFunction, DevelopmentService> developmentServiceFunction, + Function developmentOperationFunction, + BiFunction requestRefOperationFunction, + BiConsumer responseRefConsumer, + String operation) { + NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(node.getNodeType()); + AppConn appConn = AppConnManager.getAppConnManager().getAppConn(nodeInfo.getAppConnName()); + String name; + if (StringUtils.isBlank(node.getName())) { + name = node.getJobContent().get(DSSWorkFlowConstant.TITLE_KEY).toString(); + } else { + name = node.getName(); + } + if(node.getProjectId() == null || node.getProjectId() <= 0) { + DSSFlow dssFlow = dssFlowService.getFlow(node.getFlowId()); + node.setProjectId(dssFlow.getProjectID()); + } + return DevelopmentOperationUtils.tryDevelopmentOperation(() -> developmentServiceFunction.apply(appConn, node.getDssLabels()), + developmentOperationFunction, + dssJobContentRequestRef -> { + dssJobContentRequestRef.setDSSJobContent(new HashMap<>()); + String orcVersion; + try { + orcVersion = getOrcVersion(node); + } catch (Exception e) { + throw new ExternalOperationFailedException(50205, "get workflow version failed.", e); + } + if(node.getParams() != null) { + dssJobContentRequestRef.getDSSJobContent().putAll(node.getParams()); + } + dssJobContentRequestRef.getDSSJobContent().put(DSSJobContentConstant.ORC_VERSION_KEY, orcVersion); + dssJobContentRequestRef.getDSSJobContent().put(DSSJobContentConstant.ORCHESTRATION_ID, node.getFlowId()); + dssJobContentRequestRef.getDSSJobContent().put(DSSJobContentConstant.ORCHESTRATION_NAME, node.getFlowName()); + }, + refJobContentRequestRef -> { + refJobContentRequestRef.setRefJobContent(node.getJobContent()); + if (refJobContentRequestRef instanceof QueryJumpUrlRequestRef) { + ((QueryJumpUrlRequestRef) refJobContentRequestRef).setSSOUrlBuilderOperation(getSSOUrlBuilderOperation(appConn, node.getWorkspace())); + } + }, + dssContextRequestRef -> dssContextRequestRef.setContextId(node.getContextId()), + projectRefRequestRef -> { + Long refProjectId; + //todo 第一次导入时用的是dev的refProjectId +// if (node.getJobContent().containsKey(DSSWorkFlowConstant.REF_PROJECT_ID_KEY)) { +// refProjectId = DSSCommonUtils.parseToLong(node.getJobContent().get(DSSWorkFlowConstant.REF_PROJECT_ID_KEY)); +// } else { + refProjectId = parseProjectId(node.getProjectId(), appConn.getAppDesc().getAppName(), node.getDssLabels()); +// } + projectRefRequestRef.setDSSProjectId(node.getProjectId()).setRefProjectId(refProjectId).setProjectName(node.getProjectName()); + }, + (developmentOperation, developmentRequestRef) -> { + developmentRequestRef.setDSSLabels(node.getDssLabels()).setUserName(userName).setWorkspace(node.getWorkspace()).setName(name).setType(node.getNodeType()); + return requestRefOperationFunction.apply(developmentOperation, (K) developmentRequestRef); + }, responseRefConsumer, operation + " workflow node " + name); + } + + private Long parseProjectId(Long dssProjectId, String appConnName, List dssLabels) { + ProjectRelationRequest projectRelationRequest = new ProjectRelationRequest(dssProjectId, appConnName, dssLabels); + ProjectRelationResponse projectRelationResponse = (ProjectRelationResponse) projectSender.ask(projectRelationRequest); + return projectRelationResponse.getAppInstanceProjectId(); + } + + @Override + public void deleteNode(String userName, CommonAppConnNode node) throws ExternalOperationFailedException { + tryNodeOperation(userName, node, this::getRefCRUDService, developmentService -> ((RefCRUDService) developmentService).getRefDeletionOperation(), + (developmentOperation, developmentRequestRef) -> ((RefDeletionOperation) developmentOperation).deleteRef((RefJobContentRequestRef) developmentRequestRef), null, "delete"); + } + + @Override + public Map updateNode(String userName, CommonAppConnNode node) throws ExternalOperationFailedException { + tryNodeOperation(userName, node, this::getRefCRUDService, developmentService -> ((RefCRUDService) developmentService).getRefUpdateOperation(), + (developmentOperation, developmentRequestRef) -> ((RefUpdateOperation) developmentOperation).updateRef((UpdateRequestRef) developmentRequestRef), null, "update"); + return node.getJobContent(); + } + + @Override + public Map refresh(String userName, CommonAppConnNode node) { + return null; + } + + @Override + public Map copyNode(String userName, CommonAppConnNode newNode, + CommonAppConnNode oldNode, String orcVersion) throws IOException, DSSErrorException { + if (StringUtils.isBlank(orcVersion)) { + orcVersion = getOrcVersion(oldNode); + } + String finalOrcVersion = orcVersion; + RefJobContentResponseRef responseRef = tryNodeOperation(userName, oldNode, + this::getRefCRUDService, developmentService -> ((RefCRUDService) developmentService).getRefCopyOperation(), + (developmentOperation, developmentRequestRef) -> { + CopyRequestRef copyRequestRef = (CopyRequestRef) developmentRequestRef; + copyRequestRef.setNewVersion(finalOrcVersion).setName(newNode.getName()); + return ((RefCopyOperation) developmentOperation).copyRef(copyRequestRef); + }, null, "copy"); + return responseRef.getRefJobContent(); + } + + @Override + public void setNodeReadOnly(String userName, CommonAppConnNode node) { + + } + + @Override + public List listNodes(String userName, CommonAppConnNode node) { + return null; + } + + @Override + public ExportResponseRef exportNode(String userName, CommonAppConnNode node) { + return tryNodeOperation(userName, node, + (appConn, dssLabels) -> getDevelopmentService(appConn, dssLabels, DevelopmentIntegrationStandard::getRefExportService), + developmentService -> ((RefExportService) developmentService).getRefExportOperation(), + (developmentOperation, developmentRequestRef) -> + ((RefExportOperation) developmentOperation).exportRef((RefJobContentRequestRef) developmentRequestRef) + , null, "export"); + } + + @Override + public Map importNode(String userName, CommonAppConnNode node, + Supplier> getBmlResourceMap, + Supplier> getStreamResourceMap, + String orcVersion) { + RefJobContentResponseRef responseRef = tryNodeOperation(userName, node, + (appConn, dssLabels) -> getDevelopmentService(appConn, dssLabels, DevelopmentIntegrationStandard::getRefImportService), + developmentService -> ((RefImportService) developmentService).getRefImportOperation(), + (developmentOperation, developmentRequestRef) -> { + ImportRequestRef importRequestRef = (ImportRequestRef) developmentRequestRef; + if (importRequestRef.isLinkisBMLResources()) { + importRequestRef.setResourceMap(getBmlResourceMap.get()); + } else { + importRequestRef.setResourceMap(getStreamResourceMap.get()); + } + importRequestRef.setNewVersion(orcVersion).setName(node.getName()); + return ((RefImportOperation) developmentOperation).importRef(importRequestRef); + }, null, "import"); + return responseRef.getRefJobContent(); + } + + @Override + public String getNodeJumpUrl(Map params, CommonAppConnNode node, String userName) throws ExternalOperationFailedException { + ResponseRef responseRef = tryNodeOperation(userName, node, + (appConn, dssLabels) -> getDevelopmentService(appConn, dssLabels, DevelopmentIntegrationStandard::getRefQueryService), + developmentService -> ((RefQueryService) developmentService).getRefQueryOperation(), + (developmentOperation, developmentRequestRef) -> + ((RefQueryOperation) developmentOperation).query((RefJobContentRequestRef) developmentRequestRef) + , null, "query"); + if (responseRef instanceof QueryJumpUrlResponseRef) { + return ((QueryJumpUrlResponseRef) responseRef).getJumpUrl(); + } else { + throw new ExternalOperationFailedException(50025, "AppConn " + node.getNodeType() + " don't support to get jumpUrl!"); + } + } + + private String getOrcVersion(CommonAppConnNode node) throws IOException { + if (node.getFlowId() == null) { + throw new NullPointerException("The flowId is null, please ask admin for help!"); + } + DSSFlow dssFlow = dssFlowService.getFlow(node.getFlowId()); + if(StringUtils.isBlank(node.getFlowName())) { + node.setFlowName(dssFlow.getName()); + } + return workFlowParser.getValueWithKey(dssFlow.getFlowJson(), DSSJobContentConstant.ORC_VERSION_KEY); + } + + private SSOUrlBuilderOperation getSSOUrlBuilderOperation(AppConn appConn, Workspace workspace) { + if (!(appConn instanceof OnlySSOAppConn)) { + return null; + } + SSOUrlBuilderOperation ssoUrlBuilderOperation = ((OnlySSOAppConn) appConn).getOrCreateSSOStandard().getSSOBuilderService().createSSOUrlBuilderOperation(); + ssoUrlBuilderOperation.setAppName(appConn.getAppDesc().getAppName()); + SSOHelper.setSSOUrlBuilderOperation(ssoUrlBuilderOperation, workspace); + return ssoUrlBuilderOperation; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/DSSWorkflowServerApplication.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/DSSWorkflowServerApplication.scala new file mode 100644 index 000000000..bcc94524e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/DSSWorkflowServerApplication.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.{Logging, Utils} + + +object DSSWorkflowServerApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getLocalHostname + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala new file mode 100644 index 000000000..e81bf6832 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.receiver + +import com.webank.wedatasphere.dss.common.protocol.{RequestDeleteWorkflow, RequestExportWorkflow, RequestQueryWorkFlow, RequestUpdateWorkflow} +import com.webank.wedatasphere.dss.orchestrator.common.protocol._ +import com.webank.wedatasphere.dss.workflow.WorkFlowManager +import com.webank.wedatasphere.dss.workflow.common.protocol.{RequestCopyWorkflow, RequestCreateWorkflow, RequestImportWorkflow} +import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} + +import javax.annotation.PostConstruct +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +class DSSWorkflowChooser extends ReceiverChooser { + + + @Autowired + var workflowManager: WorkFlowManager = _ + + var receiver: Option[DSSWorkflowReceiver] = _ + + @PostConstruct + def init(): Unit = receiver = Some(new DSSWorkflowReceiver(workflowManager)) + + override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = event.message match { + case _: RequestCreateWorkflow => receiver + case _: RequestUpdateWorkflow => receiver + case _: RequestDeleteWorkflow => receiver + case _: RequestExportWorkflow => receiver + case _: RequestImportWorkflow => receiver + case _: RequestCopyWorkflow => receiver + case _: RequestQueryWorkFlow => receiver + case _: RequestConvertOrchestrations => receiver + case _ => None + } +} \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala new file mode 100644 index 000000000..7fa001d2c --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala @@ -0,0 +1,109 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.receiver + +import java.util + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException +import com.webank.wedatasphere.dss.common.protocol._ +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestConvertOrchestrations +import com.webank.wedatasphere.dss.standard.app.sso.Workspace +import com.webank.wedatasphere.dss.workflow.WorkFlowManager +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow +import com.webank.wedatasphere.dss.workflow.common.protocol._ +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam +import org.apache.linkis.rpc.{Receiver, Sender} + +import scala.concurrent.duration.Duration + +class DSSWorkflowReceiver(workflowManager: WorkFlowManager) extends Receiver { + override def receive(message: Any, sender: Sender): Unit = {} + + override def receiveAndReply(message: Any, sender: Sender): Any = message match { + + case reqCreateFlow: RequestCreateWorkflow => + val dssFlow = workflowManager.createWorkflow(reqCreateFlow.getUserName, reqCreateFlow.getProjectId, reqCreateFlow.getWorkflowName, + reqCreateFlow.getContextIDStr, reqCreateFlow.getDescription, + reqCreateFlow.getParentFlowID, reqCreateFlow.getUses, + reqCreateFlow.getLinkedAppConnNames, reqCreateFlow.getDssLabels, reqCreateFlow.getOrcVersion, + reqCreateFlow.getSchedulerAppConnName) + val responseCreateWorkflow = new ResponseCreateWorkflow() + responseCreateWorkflow.setDssFlow(dssFlow) + responseCreateWorkflow + + case reqUpdateFlow: RequestUpdateWorkflow => + workflowManager.updateWorkflow(reqUpdateFlow.userName, + reqUpdateFlow.flowID, reqUpdateFlow.flowName, + reqUpdateFlow.description, reqUpdateFlow.uses) + new ResponseUpdateWorkflow(JobStatus.Success) + + case reqDeleteFlow: RequestDeleteWorkflow => + workflowManager.deleteWorkflow(reqDeleteFlow.userName, reqDeleteFlow.flowID) + new ResponseDeleteWorkflow(JobStatus.Success) + + case reqExportFlow: RequestExportWorkflow => + val dssExportFlowResource: util.Map[String, AnyRef] = workflowManager.exportWorkflow( + reqExportFlow.userName, + reqExportFlow.flowID, + reqExportFlow.projectId, + reqExportFlow.projectName, + DSSCommonUtils.COMMON_GSON.fromJson(reqExportFlow.workspaceStr, classOf[Workspace]), + reqExportFlow.dssLabelList) + ResponseExportWorkflow(dssExportFlowResource.get("resourceId").toString, dssExportFlowResource.get("version").toString, + reqExportFlow.flowID) + + case requestImportWorkflow: RequestImportWorkflow => + val dssFlowImportParam: DSSFlowImportParam = new DSSFlowImportParam() + dssFlowImportParam.setProjectID(requestImportWorkflow.getProjectId) + dssFlowImportParam.setProjectName(requestImportWorkflow.getProjectName) + dssFlowImportParam.setUserName(requestImportWorkflow.getUserName) + dssFlowImportParam.setOrcVersion(requestImportWorkflow.getOrcVersion) + dssFlowImportParam.setWorkspace(requestImportWorkflow.getWorkspace) + dssFlowImportParam.setContextId(requestImportWorkflow.getContextId) + val dssFlows = workflowManager.importWorkflow(requestImportWorkflow.getUserName, + requestImportWorkflow.getResourceId, + requestImportWorkflow.getBmlVersion, + dssFlowImportParam, requestImportWorkflow.getDssLabels) + import scala.collection.JavaConversions._ + val dssFlowIds = dssFlows.map(dssFlow => (dssFlow.getId, dssFlow.getFlowJson)).toMap + new ResponseImportWorkflow(JobStatus.Success, dssFlowIds) + + case requestCopyWorkflow: RequestCopyWorkflow => + val copyFlow: DSSFlow = workflowManager.copyRootFlowWithSubFlows(requestCopyWorkflow.getUserName, + requestCopyWorkflow.getRootFlowId, + requestCopyWorkflow.getWorkspace, + requestCopyWorkflow.getProjectName, + requestCopyWorkflow.getContextIdStr, + requestCopyWorkflow.getOrcVersion, + requestCopyWorkflow.getDescription, + requestCopyWorkflow.getDssLabels) + new ResponseCopyWorkflow(copyFlow) + + case requestQueryWorkFlow: RequestQueryWorkFlow => + val dssFlow: DSSFlow = workflowManager.queryWorkflow(requestQueryWorkFlow.userName, + requestQueryWorkFlow.rootFlowId) + new ResponseQueryWorkflow(dssFlow) + + case requestConvertOrchestrator: RequestConvertOrchestrations => + workflowManager.convertWorkflow(requestConvertOrchestrator) + + case _ => throw new DSSErrorException(90000, "Not support protocol " + message) + } + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/service/BMLService.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/service/BMLService.scala new file mode 100644 index 000000000..4a057bbea --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/service/BMLService.scala @@ -0,0 +1,203 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.service + +import java.io.{ByteArrayInputStream, InputStream} +import java.util +import java.util.UUID + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException +import com.webank.wedatasphere.dss.common.utils.IoUtils +import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory} +import org.apache.linkis.bml.protocol.{BmlDownloadResponse, BmlUpdateResponse, BmlUploadResponse} +import org.apache.linkis.common.utils.{JavaLog, Utils} +import org.apache.commons.io.IOUtils +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +@Component +class BMLService extends JavaLog { + + var bmlClient: BmlClient = _ + + def getBmlClient(userName: String): BmlClient = { + if (bmlClient == null) synchronized { + if (bmlClient == null) { + bmlClient = BmlClientFactory.createBmlClient(userName) + } + } + bmlClient + } + + def upload(userName: String, content: String, fileName: String, projectName: String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = getBmlClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def upload(userName: String, inputStream: InputStream, fileName: String, projectName: String): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def update(userName: String, resourceId: String, inputStream: InputStream): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, "", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def update(userName: String, resourceId: String, content: String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = getBmlClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, UUID.randomUUID().toString + ".json", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def query(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "string" -> inputstremToString(resource.inputStream) + } + + def download(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "is" -> resource.inputStream + } + + def downloadToLocalPath(userName: String, resourceId: String, version: String, path: String): String = { + val result = download(userName, resourceId, version) + val is = result.get("is").asInstanceOf[InputStream] + val os = IoUtils.generateExportOutputStream(path) + Utils.tryFinally(IOUtils.copy(is, os)) { + IOUtils.closeQuietly(os) + IOUtils.closeQuietly(is) + } + path + } + + def downloadAndGetFlowJson(userName: String, resourceId: String, version: String, path: String): String = { + downloadToLocalPath(userName, resourceId, version, path) + //因为下载到指定目录后返回的resource的Stream为null,只能从文件重新读取。 + val is = IoUtils.generateInputInputStream(path) + Utils.tryFinally(inputstremToString(is))(IOUtils.closeQuietly(is)) + } + + def readLocalResourceFile(userName: String, readPath: String): InputStream = { + IoUtils.generateInputInputStream(readPath) + } + + def readLocalFlowJsonFile(userName: String, readPath: String): String = { + var inputStream: InputStream = null + var flowJson: String = null + Utils.tryFinally { + inputStream = IoUtils.generateInputInputStream(readPath) + flowJson = inputstremToString(inputStream) + } { + IOUtils.closeQuietly(inputStream) + } + flowJson + } + + + def readFlowJsonFromBML(userName: String, resourceId: String, version: String): String = { + + val result = download(userName, resourceId, version) + val is = result.get("is").asInstanceOf[InputStream] + + var inputStream: InputStream = null + var flowJson: String = null + Utils.tryFinally { + inputStream = is + flowJson = inputstremToString(inputStream) + } { + IOUtils.closeQuietly(inputStream) + } + flowJson + } + + + private def inputstremToString(inputStream: InputStream): String = { + scala.io.Source.fromInputStream(inputStream).mkString + } + + def createBmlProject(username: String, projectName: String, editUsers: util.List[String], + accessUsers: util.List[String]): Unit = { + val client = getBmlClient(username) + val response = client.createBmlProject(username, projectName, accessUsers, editUsers) + if (response.isSuccess) { + logger.info(s"for user $username create bml project $projectName success") + } else { + logger.error(s"for user $username create bml project $projectName failed") + } + } + + def attachResourceAndProject(username: String, projectName: String, resourceId: String): Unit = { + val client = getBmlClient(username) + val response = client.attachResourceAndProject(projectName, resourceId) + if (response.isSuccess) { + logger.info(s"attach $username and $projectName success") + } else { + logger.error(s"attach $username and $projectName failed") + } + } + + + def updateProjectPriv(projectName: String, username: String, editUsers: util.List[String], + accessUsers: util.List[String]): Unit = { + val client = getBmlClient(username) + val response = client.updateProjectPriv(username, projectName, editUsers, accessUsers) + if (response.isSuccess) { + logger.info(s"attach $username and $projectName success") + } else { + logger.error(s"attach $username and $projectName failed") + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/pom.xml new file mode 100644 index 000000000..8fab7b78e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/pom.xml @@ -0,0 +1,40 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-workflow + pom + + + dss-workflow-common + dss-linkis-node-execution + dss-flow-execution-server + dss-workflow-server + dss-workflow-sdk + dss-workflow-conversion-standard + + + \ No newline at end of file diff --git a/dss-orchestrator/pom.xml b/dss-orchestrator/pom.xml new file mode 100644 index 000000000..599f996be --- /dev/null +++ b/dss-orchestrator/pom.xml @@ -0,0 +1,40 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-orchestrator + pom + + + dss-orchestrator-common + dss-orchestrator-core + dss-orchestrator-db + dss-orchestrator-loader + orchestrators/dss-workflow + dss-orchestrator-conversion-standard + + + diff --git a/dss-standard/development-standard/development-process-standard-execution/pom.xml b/dss-standard/development-standard/development-process-standard-execution/pom.xml new file mode 100644 index 000000000..ed02aa03b --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/pom.xml @@ -0,0 +1,76 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-development-process-standard-execution + + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + net.alchim31.maven + scala-maven-plugin + + + + + \ No newline at end of file diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionListener.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionListener.java new file mode 100644 index 000000000..3614f0f08 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionListener.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.async; + + +public interface RefExecutionListener { +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionResponseListener.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionResponseListener.java new file mode 100644 index 000000000..c2716b7d9 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionResponseListener.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.async; + +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef; + + +public interface RefExecutionResponseListener extends RefExecutionListener { + + void onRefExecutionCompleted(ExecutionResponseRef response); + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionStatusListener.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionStatusListener.java new file mode 100644 index 000000000..7a89bc4f4 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/async/RefExecutionStatusListener.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.async; + +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.AsyncExecutionResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; + + +public interface RefExecutionStatusListener> extends RefExecutionListener { + + void beforeSubmit(K requestRef); + + void afterSubmit(K requestRef, RefExecutionAction action); + + void afterAsyncResponseRef(AsyncExecutionResponseRef response); + + void afterCompletedExecutionResponseRef(K requestRef, RefExecutionAction action, + ExecutionResponseRef response); + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/AbstractRefExecutionAction.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/AbstractRefExecutionAction.java new file mode 100644 index 000000000..187f94e16 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/AbstractRefExecutionAction.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.common; + +import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext; + + +public abstract class AbstractRefExecutionAction implements RefExecutionAction, LongTermRefExecutionAction { + + private String id; + private ExecutionRequestRefContext executionRequestRefContext; + private RefExecutionState state; + private int schedulerId; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public boolean isKilledFlag() { + return killedFlag; + } + + public void setKilledFlag(boolean killedFlag) { + this.killedFlag = killedFlag; + } + + private boolean killedFlag = false; + + @Override + public ExecutionRequestRefContext getExecutionRequestRefContext() { + return executionRequestRefContext; + } + + public void setExecutionRequestRefContext(ExecutionRequestRefContext executionRequestRefContext) { + this.executionRequestRefContext = executionRequestRefContext; + } + + public RefExecutionState getState() { + return state; + } + + public void setState(RefExecutionState state) { + this.state = state; + } + + @Override + public int getSchedulerId() { + return schedulerId; + } + + @Override + public void setSchedulerId(int schedulerId) { + this.schedulerId = schedulerId; + } +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/LongTermRefExecutionAction.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/LongTermRefExecutionAction.java new file mode 100644 index 000000000..2f8cc3824 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/LongTermRefExecutionAction.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.common; + + +public interface LongTermRefExecutionAction extends RefExecutionAction { + + void setSchedulerId(int schedulerId); + + int getSchedulerId(); + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/RefExecutionAction.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/RefExecutionAction.java new file mode 100644 index 000000000..b88677e0f --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/RefExecutionAction.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.common; + +import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext; + + +public interface RefExecutionAction { + + ExecutionRequestRefContext getExecutionRequestRefContext(); + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/RefExecutionState.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/RefExecutionState.java new file mode 100644 index 000000000..27dc8eb49 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/common/RefExecutionState.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.common; + + +public enum RefExecutionState { + /** + * NodeExecution的状态枚举 + */ + Accepted(0, "Accepted"), + Running(1, "Running"), + Success(2, "Success"), + Failed(3, "Failed"), + Killed(4, "Killed"), + Alert(5, "Alert"); + + private int code; + private String status; + + private RefExecutionState(int code, String status){ + this.code = code; + this.status = status; + } + + public int getCode() { + return code; + } + + public String getStatus() { + return status; + } + + public static boolean isCompleted(RefExecutionState state){ + return Success.equals(state) || Failed.equals(state) || Killed.equals(state); + } + + public static boolean isCompleted(String state){ + return Success.status.equals(state) || + Failed.status.equals(state) || + Killed.status.equals(state); + } + + public boolean isCompleted(){ + return isCompleted(this); + } + + public boolean isSuccess(){ + return Success.equals(this); + } + + + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/CallbackLongTermRefExecutionOperation.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/CallbackLongTermRefExecutionOperation.java new file mode 100644 index 000000000..ab476f5d4 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/CallbackLongTermRefExecutionOperation.java @@ -0,0 +1,57 @@ + +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.core; + +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; +import com.webank.wedatasphere.dss.standard.app.development.listener.conf.RefExecutionConfiguration; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.AsyncExecutionResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + + +public abstract class CallbackLongTermRefExecutionOperation> + extends LongTermRefExecutionOperation { + + protected Map asyncResponses = new ConcurrentHashMap(); + + public String getCallbackURL(ExecutionRequestRefContext executionRequestRefContext){ + String urlSuffix = RefExecutionConfiguration.CALL_BACK_URL().getValue(); + return executionRequestRefContext.getGatewayUrl() + urlSuffix; + } + + public abstract void acceptCallback(Map callbackMap); + + protected void markCompleted(RefExecutionAction action, ExecutionResponseRef resultResponse) { + AsyncExecutionResponseRef response = asyncResponses.get(action); + asyncResponses.remove(action); + response.setCompleted(resultResponse); + } + + @Override + protected AsyncExecutionResponseRef createAsyncResponseRef(K requestRef, RefExecutionAction action) { + AsyncExecutionResponseRef oldResponse = super.createAsyncResponseRef(requestRef, action); + AsyncExecutionResponseRef response = AsyncExecutionResponseRef.newBuilder().setAsyncExecutionResponseRef(oldResponse) + .setAskStatePeriod(RefExecutionConfiguration.CALLBACK_REF_EXECUTION_REFRESH_INTERVAL().getValue().toLong()) + .build(); + asyncResponses.put(action, response); + return response; + } +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/ExecutionRequestRefContext.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/ExecutionRequestRefContext.java new file mode 100644 index 000000000..3c7b9a45f --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/ExecutionRequestRefContext.java @@ -0,0 +1,178 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.core; + +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.resultset.ResultSet; +import org.apache.linkis.common.io.resultset.ResultSetReader; +import org.apache.linkis.common.io.resultset.ResultSetWriter; + +import java.util.Map; + + +public interface ExecutionRequestRefContext { + + /** + * 包含了 DSS 工作流节点右侧属性栏的所有信息,与 {@code RefJobContentRequestRef} 中的 getRefJobContent() + * 返回的内容完全相同。 + * 该方法是一个历史遗留方法,新的 AppConn 在实现时推荐直接使用 {@code RefJobContentRequestRef} 中的 getRefJobContent() + * @return 包含了 DSS 工作流节点右侧属性栏所有信息的 Map + */ + @Deprecated + Map getRuntimeMap(); + + /** + * 往客户端推送实时日志,该方法与 {@code Logger.info()} 效果等同,都会被 Linkis EngineConn 框架推送回客户端, + * 但是 Linkis EngineConn 在给 Entrance 推送日志时,会过滤掉一些无用日志,通过日志框架 {@code Logger} 打印的 + * 日志,一部分会被过滤规则识别并过滤掉,不会给 Entrance 和客户端推送,但是通过该方法推送的日志不会被过滤。 + * @param log 日志 + */ + void appendLog(String log); + + /** + * 实时更新第三方 AppConn refJob 的执行进度 + * @param progress 进度条 + */ + void updateProgress(float progress); + + /** + * 获取指定 Linkis Job 的基本信息。 + * 该方法通常是提供给第三方节点想要使用上游的工作流节点内容时使用 + * @param jobId Linkis Job Id + * @return Linkis Job 基本信息 + */ + LinkisJob fetchLinkisJob(long jobId); + + /** + * 获取指定 Linkis Job 的所有的结果集路径。 + * 该方法通常是提供给第三方节点想要使用上游的工作流节点内容时使用 + */ + FsPath[] fetchLinkisJobResultSetPaths(long jobId); + + /** + * 获取节点的实际执行用户。 + * 例如:如果工作流的提交调度用户为 enjoyyin,而调度系统设置的代理执行用户为 hadoop, 则该方法返回的是 hadoop。 + * @return 执行用户 + */ + String getUser(); + + /** + * 获取节点的提交用户。 + * 例如:如果工作流的提交调度用户为 enjoyyin,而调度系统设置的代理执行用户为 hadoop, 则该方法返回的是 enjoyyin。 + * @return 提交用户 + */ + String getSubmitUser(); + + /** + * 创建一个数据表类型的结果集 Writer 类,用于将第三方节点产生的结果集作为一张表格写入到 Linkis 之中。 + * @param 一般为 TableMetaData + * @param 一般为 TableRecord + * @return 数据表类型的结果集 Writer 类 + */ + ResultSetWriter createTableResultSetWriter(); + + /** + * 创建一个数据表格式类型的结果集 Writer 类,用于将第三方节点产生的结果集作为一张表格写入到 Linkis 之中。 + * @param resultSetAlias 结果集文件的文件名前缀,如果为空,则等同于 {@code createTableResultSetWriter()} + * @param 一般为 TableMetaData + * @param 一般为 TableRecord + * @return 数据表类型的结果集 Writer 类 + */ + ResultSetWriter createTableResultSetWriter(String resultSetAlias); + + /** + * 创建一个文本格式类型的结果集 Writer 类,用于将第三方节点产生的结果集作为一个text文件写入到 Linkis 之中。 + * @param 一般为 LineMetaData + * @param 一般为 LineRecord + * @return 文本格式类型的结果集 Writer 类 + */ + ResultSetWriter createTextResultSetWriter(); + + /** + * 创建一个文本格式类型的结果集 Writer 类,用于将第三方节点产生的结果集作为一个text文件写入到 Linkis 之中。 + * @param resultSetAlias 结果集文件的文件名前缀,如果为空,则等同于 {@code createTextResultSetWriter()} + * @param 一般为 LineMetaData + * @param 一般为 LineRecord + * @return 文本格式类型的结果集 Writer 类 + */ + ResultSetWriter createTextResultSetWriter(String resultSetAlias); + + /** + * 创建一个 HTML 格式类型的结果集 Writer 类,用于将第三方节点产生的结果集作为一个 HTML 文件写入到 Linkis 之中。 + * @param 一般为 LineMetaData + * @param 一般为 LineRecord + * @return HTML 格式类型的结果集 Writer 类 + */ + ResultSetWriter createHTMLResultSetWriter(); + + /** + * 创建一个 HTML 格式类型的结果集 Writer 类,用于将第三方节点产生的结果集作为一个 HTML 文件写入到 Linkis 之中。 + * @param resultSetAlias 结果集文件的文件名前缀,如果为空,则等同于 {@code createHTMLResultSetWriter()} + * @param 一般为 LineMetaData + * @param 一般为 LineRecord + * @return HTML 格式类型的结果集 Writer 类 + */ + ResultSetWriter createHTMLResultSetWriter(String resultSetAlias); + + /** + * 创建一个图片格式类型的结果集 Writer 类,用于将第三方节点产生的结果集作为一个图片文件写入到 Linkis 之中。 + * @param 一般为 LineMetaData + * @param 一般为 LineRecord + * @return 图片格式类型的结果集 Writer 类 + */ + ResultSetWriter createPictureResultSetWriter(); + + /** + * 创建一个图片格式类型的结果集 Writer 类,用于将第三方节点产生的结果集作为一个图片文件写入到 Linkis 之中。 + * @param resultSetAlias 结果集文件的文件名前缀,如果为空,则等同于 {@code createPictureResultSetWriter()} + * @param 一般为 LineMetaData + * @param 一般为 LineRecord + * @return 图片格式类型的结果集 Writer 类 + */ + ResultSetWriter createPictureResultSetWriter(String resultSetAlias); + + ResultSetWriter createResultSetWriter(ResultSet resultSet, + String resultSetAlias); + + /** + * 通过传入一个结果集文件路径,获取一个可以读取结果集内容的 Reader。 + * 根据结果集类型的不同,返回的 {@code ResultSetReader} 中,其 泛型 M 和 泛型 R 也不相同。 + * 如果结果集是 数据表格式类型,则泛型 M 为 TableMetaData,泛型 R 为 TableRecord;其他结果集格式类型的,目前 + * 泛型 M 都是 LineMetaData,泛型 R 为 LineRecord。
+ * @param fsPath 结果集文件路径 + * @param 如果结果集是 数据表格式类型,则泛型 M 为 TableMetaData;其他结果集格式类型的泛型 M 都是 LineMetaData + * @param 如果结果集是 数据表格式类型,则泛型 R 为 TableRecord;其他结果集格式类型的泛型 R 都是 LineRecord + * @return 可以读取结果集内容的 Reader + */ + ResultSetReader getResultSetReader(FsPath fsPath); + + /** + * 该方法用于将第三方节点写入缓存的一个结果集文件,持久化并发送给Linkis Entrance。 + * 如果您调用了 createResultSetWriter 等方法,请在最后调用此方法。 + * @param resultSetWriter 需要持久化的结果集 Writer + */ + void sendResultSet(ResultSetWriter resultSetWriter); + + /** + * Linkis Gateway 的 base URL + * @return Linkis Gateway 的 base URL + */ + String getGatewayUrl(); + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/Killable.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/Killable.java new file mode 100644 index 000000000..4a871c2db --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/Killable.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.core; + + +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; + + +public interface Killable { + public boolean kill(RefExecutionAction action); +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/LinkisJob.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/LinkisJob.java new file mode 100644 index 000000000..1889733a6 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/LinkisJob.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.core; + +import org.apache.linkis.manager.label.entity.Label; +import java.util.List; +import java.util.Map; + + +public interface LinkisJob { + + String getResultLocation(); + + String getSubmitUser(); + + String getExecuteUser(); + + String getStatus(); + + Map getSource(); + + Map getParams(); + + List> getLabels(); + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/LongTermRefExecutionOperation.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/LongTermRefExecutionOperation.java new file mode 100644 index 000000000..69568a7d7 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/LongTermRefExecutionOperation.java @@ -0,0 +1,119 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.core; + +import com.webank.wedatasphere.dss.standard.app.development.listener.async.RefExecutionStatusListener; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.AbstractRefExecutionAction; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.AsyncExecutionResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.listener.scheduler.LongTermRefExecutionScheduler; +import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +import java.util.ArrayList; +import java.util.List; + +/** + * Description: LongTermRefExecutionOperation is used to execute long-term tasks in external AppConn systems. + * Long-term task usually supports to execute asyncly and can fetch status and logs when it is submitted. + * + */ +public abstract class LongTermRefExecutionOperation> + extends AbstractDevelopmentOperation implements RefExecutionOperation { + + private List> refExecutionListener = new ArrayList<>(); + private LongTermRefExecutionScheduler scheduler = SchedulerManager.getScheduler(); + + public void addRefExecutionStatusListener(RefExecutionStatusListener refExecutionListener) { + for(RefExecutionStatusListener listener : this.refExecutionListener){ + if (listener.getClass().equals(refExecutionListener.getClass())){ + return; + } + } + this.refExecutionListener.add(refExecutionListener); + } + + /** + * 异步提交给第三方 AppConn,请求执行该 refJob。 + * 通常情况下,第三方 AppConn 会返回一个 executionId,请将该 executionId 放入 {@code RefExecutionAction}之中, + * 以便后面的 {@code state()} 和 {@code result()} 方法可通过该 {@code RefExecutionAction} 中的 executionId获取 + * 到这个 refJob 的状态和结果。 + * @param requestRef 包含了第三方 refJob 信息的 requestRef + * @return 包含了 executionId 的 RefExecutionAction + * @throws ExternalOperationFailedException 提交失败时抛出该异常 + */ + protected abstract RefExecutionAction submit(K requestRef) throws ExternalOperationFailedException; + + /** + * 获取已提交给第三方 AppConn 的 refJob 的状态。 + * 您也可以在该方法内,既获取 refJob 的状态,同时也获取 refJob 的执行进度,然后通过调用 + * {@code RefExecutionAction.getExecutionRequestRefContext().updateProgress()} 方法更新该 refJob 的执行进度, + * 以便 DSS 能够直接在前端展示出该 refJob 的实时进度信息。 + * @param action 包含了 executionId 的 RefExecutionAction + * @return 返回 refJob 的状态 + * @throws ExternalOperationFailedException 获取状态失败时抛出该异常 + */ + public abstract RefExecutionState state(RefExecutionAction action) throws ExternalOperationFailedException; + + /** + * 获取已提交给第三方 AppConn 的 refJob 的结果。 + * 请注意:该方法只在 {@code state()} 方法返回的状态为完成时才会被调用。 + * @param action 包含了 executionId 的 RefExecutionAction + * @return 返回 refJob 的结果 + * @throws ExternalOperationFailedException 获取结果失败时抛出该异常 + */ + public abstract ExecutionResponseRef result(RefExecutionAction action) throws ExternalOperationFailedException; + + protected ExecutionRequestRefContext createExecutionRequestRefContext(K requestRef) { + return requestRef.getExecutionRequestRefContext(); + } + + @Override + public final ResponseRef execute(K requestRef) throws ExternalOperationFailedException { + refExecutionListener.forEach(l -> l.beforeSubmit(requestRef)); + RefExecutionAction action = submit(requestRef); + if(action instanceof AbstractRefExecutionAction) { + ((AbstractRefExecutionAction) action).setExecutionRequestRefContext(createExecutionRequestRefContext(requestRef)); + } + refExecutionListener.forEach(l -> l.afterSubmit(requestRef, action)); + RefExecutionState state = state(action); + if(state != null && state.isCompleted()) { + ExecutionResponseRef response = result(action); + refExecutionListener.forEach(l -> l.afterCompletedExecutionResponseRef(requestRef, action, response)); + return response; + } else { + AsyncExecutionResponseRef oldResponse = createAsyncResponseRef(requestRef, action); + AsyncExecutionResponseRef response = AsyncExecutionResponseRef.newBuilder().setAsyncExecutionResponseRef(oldResponse) + .setRefExecutionOperation(this) + .addListener(r -> refExecutionListener.forEach(l -> l.afterCompletedExecutionResponseRef(requestRef, action, r))).build(); + refExecutionListener.forEach(l -> l.afterAsyncResponseRef(response)); + scheduler.addAsyncResponse(response); + return response; + } + } + + protected AsyncExecutionResponseRef createAsyncResponseRef(K requestRef, RefExecutionAction action) { + return AsyncExecutionResponseRef.newBuilder().setAction(action) + .setExecutionRequestRef(requestRef).build(); + } + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/ParamsNode.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/ParamsNode.java new file mode 100644 index 000000000..ddd972b0a --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/ParamsNode.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.core; + +import java.util.Map; + + +public interface ParamsNode { + public Map getParams(); +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/Procedure.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/Procedure.java new file mode 100644 index 000000000..53e21a08d --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/Procedure.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.core; + + +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; + + +public interface Procedure { + + public float progress(RefExecutionAction action); + + public String log(RefExecutionAction action); + + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/SchedulerManager.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/SchedulerManager.java new file mode 100644 index 000000000..3c457fec5 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/core/SchedulerManager.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.core; + +import com.webank.wedatasphere.dss.standard.app.development.listener.scheduler.ListenerEventBusRefExecutionScheduler; +import com.webank.wedatasphere.dss.standard.app.development.listener.scheduler.LongTermRefExecutionScheduler; + + +public class SchedulerManager { + + private static LongTermRefExecutionScheduler scheduler = new ListenerEventBusRefExecutionScheduler(); + + static { + scheduler.start(); + } + + public static LongTermRefExecutionScheduler getScheduler(){ + return scheduler; + } +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/exception/AppConnExecutionErrorException.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/exception/AppConnExecutionErrorException.java new file mode 100644 index 000000000..c313d9070 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/exception/AppConnExecutionErrorException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class AppConnExecutionErrorException extends ErrorException { + + public AppConnExecutionErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public AppConnExecutionErrorException(int errCode, String desc, Throwable cause) { + super(errCode, desc); + initCause(cause); + } +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/exception/AppConnExecutionWarnException.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/exception/AppConnExecutionWarnException.java new file mode 100644 index 000000000..c65118718 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/exception/AppConnExecutionWarnException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.exception; + +import org.apache.linkis.common.exception.WarnException; + + +public class AppConnExecutionWarnException extends WarnException { + + public AppConnExecutionWarnException(int errCode, String desc) { + super(errCode, desc); + } + + public AppConnExecutionWarnException(int errCode, String desc, Throwable cause) { + super(errCode, desc); + initCause(cause); + } +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/AsyncExecutionResponseRef.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/AsyncExecutionResponseRef.java new file mode 100644 index 000000000..6f3f9ec65 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/AsyncExecutionResponseRef.java @@ -0,0 +1,229 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.ref; + +import com.webank.wedatasphere.dss.standard.app.development.listener.async.RefExecutionResponseListener; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; +import com.webank.wedatasphere.dss.standard.app.development.listener.core.LongTermRefExecutionOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.AsyncResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + + +public interface AsyncExecutionResponseRef extends AsyncResponseRef { + + RefExecutionAction getAction(); + + RefExecutionRequestRef getExecutionRequestRef(); + + LongTermRefExecutionOperation getRefExecutionOperation(); + + long getAskStatePeriod(); + + void setCompleted(ExecutionResponseRef response); + + long getMaxLoopTime(); + + static Builder newBuilder() { + return new Builder(); + } + + class Builder { + + private final AsyncExecutionResponseRefImpl asyncResponseRef = new AsyncExecutionResponseRefImpl(); + + public Builder setAction(RefExecutionAction action) { + this.asyncResponseRef.action = action; + return this; + } + + public Builder addListener(RefExecutionResponseListener listener) { + this.asyncResponseRef.listeners.add(listener); + return this; + } + + public Builder setRefExecutionOperation(LongTermRefExecutionOperation refExecutionOperation) { + this.asyncResponseRef.refExecutionOperation = refExecutionOperation; + return this; + } + + public Builder setExecutionRequestRef(RefExecutionRequestRef requestRef) { + this.asyncResponseRef.requestRef = requestRef; + return this; + } + + public Builder setMaxLoopTime(long maxLoopTime) { + this.asyncResponseRef.maxLoopTime = maxLoopTime; + return this; + } + + public Builder setAskStatePeriod(long askStatePeriod) { + this.asyncResponseRef.askStatePeriod = askStatePeriod; + return this; + } + + public Builder setAsyncExecutionResponseRef(AsyncExecutionResponseRef responseRef) { + if(!(responseRef instanceof AsyncExecutionResponseRefImpl)) { + return this; + } + AsyncExecutionResponseRefImpl asyncResponseRef = (AsyncExecutionResponseRefImpl) responseRef; + if(this.asyncResponseRef.action == null) { + this.asyncResponseRef.action = asyncResponseRef.action; + } + if(!asyncResponseRef.listeners.isEmpty()) { + this.asyncResponseRef.listeners.addAll(asyncResponseRef.listeners); + } + if(this.asyncResponseRef.refExecutionOperation == null) { + this.asyncResponseRef.refExecutionOperation = asyncResponseRef.refExecutionOperation; + } + if(this.asyncResponseRef.requestRef == null) { + this.asyncResponseRef.requestRef = asyncResponseRef.requestRef; + } + if(this.asyncResponseRef.maxLoopTime <= 0) { + this.asyncResponseRef.maxLoopTime = asyncResponseRef.maxLoopTime; + } + if(this.asyncResponseRef.askStatePeriod == 1000) { + this.asyncResponseRef.askStatePeriod = asyncResponseRef.askStatePeriod; + } + if(this.asyncResponseRef.response == null) { + this.asyncResponseRef.response = asyncResponseRef.response; + } + this.asyncResponseRef.startTime = asyncResponseRef.startTime; + if(!this.asyncResponseRef.isCompleted) { + this.asyncResponseRef.isCompleted = asyncResponseRef.isCompleted; + } + return this; + } + + public AsyncExecutionResponseRef build() { + return this.asyncResponseRef; + } + + class AsyncExecutionResponseRefImpl extends ResponseRefImpl implements AsyncExecutionResponseRef { + + RefExecutionAction action; + List listeners = new ArrayList<>(); + LongTermRefExecutionOperation refExecutionOperation; + RefExecutionRequestRef requestRef; + ExecutionResponseRef response; + long maxLoopTime; + long askStatePeriod = 1000; + long startTime = System.currentTimeMillis(); + boolean isCompleted = false; + private final Object lock = new Object(); + + public AsyncExecutionResponseRefImpl() { + super("", 1, "", new HashMap<>(0)); + } + + @Override + public RefExecutionAction getAction() { + return action; + } + + @Override + public RefExecutionRequestRef getExecutionRequestRef() { + return requestRef; + } + + @Override + public LongTermRefExecutionOperation getRefExecutionOperation() { + return refExecutionOperation; + } + + @Override + public long getAskStatePeriod() { + return askStatePeriod; + } + + @Override + public void setCompleted(ExecutionResponseRef response) { + isCompleted = true; + this.response = response; + status = response.getStatus(); + responseBody = response.getResponseBody(); + synchronized (lock) { + lock.notifyAll(); + } + listeners.forEach(l -> l.onRefExecutionCompleted(response)); + } + + @Override + public long getMaxLoopTime() { + return maxLoopTime; + } + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public ResponseRef getResponse() { + return response; + } + + @Override + public void waitForCompleted() throws InterruptedException { + synchronized (lock) { + while(!isCompleted) { + lock.wait(2000); + } + } + } + + @Override + public void notifyMe(Consumer notifyListener) { + addListener(notifyListener::accept); + } + + @Override + public Map toMap() { + if(response != null) { + return response.toMap(); + } + return super.toMap(); + } + + @Override + public String getErrorMsg() { + if(response != null) { + return response.getErrorMsg(); + } + return super.errorMsg; + } + + @Override + public boolean isSucceed() { + return response != null && response.isSucceed(); + } + + @Override + public boolean isFailed() { + return response != null && response.isFailed(); + } + } + + } + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/ExecutionResponseRef.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/ExecutionResponseRef.java new file mode 100644 index 000000000..4de1cfc5b --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/ExecutionResponseRef.java @@ -0,0 +1,63 @@ +package com.webank.wedatasphere.dss.standard.app.development.listener.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +/** + * @author enjoyyin + * @date 2022-03-11 + * @since 0.5.0 + */ +public interface ExecutionResponseRef extends ResponseRef { + + Throwable getException(); + + static ExecutionResponseRefBuilder newBuilder() { + return new ExecutionResponseRefBuilder(); + } + + class ExecutionResponseRefBuilder extends ResponseRefBuilder.ExternalResponseRefBuilder { + + private Throwable exception; + + public ExecutionResponseRefBuilder setException(Throwable exception) { + this.exception = exception; + error(exception); + return this; + } + + public ExecutionResponseRefBuilder setResponseRef(ResponseRef responseRef) { + status = responseRef.getStatus(); + errorMsg = responseRef.getErrorMsg(); + responseMap = responseRef.toMap(); + responseBody = responseRef.getResponseBody(); + return this; + } + + public ExecutionResponseRef error() { + if(exception != null) { + return build(); + } else { + return error("Unknown reason, Please ask admin for help."); + } + } + + class ExecutionResponseRefImpl extends ResponseRefImpl implements ExecutionResponseRef { + public ExecutionResponseRefImpl() { + super(ExecutionResponseRefBuilder.this.responseBody, ExecutionResponseRefBuilder.this.status, + ExecutionResponseRefBuilder.this.errorMsg, ExecutionResponseRefBuilder.this.responseMap); + } + @Override + public Throwable getException() { + return exception; + } + } + + @Override + protected ExecutionResponseRef createResponseRef() { + return new ExecutionResponseRefImpl(); + } + } + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/RefExecutionRequestRef.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/RefExecutionRequestRef.java new file mode 100644 index 000000000..f1a5ec3fc --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/ref/RefExecutionRequestRef.java @@ -0,0 +1,105 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.ref; + +import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext; +import com.webank.wedatasphere.dss.standard.app.development.ref.DSSContextRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.ProjectRefRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.DevelopmentRequestRefImpl; + +import java.util.Map; + + +public interface RefExecutionRequestRef> + extends RefJobContentRequestRef { + + /** + * 节点执行的上下文信息类,详见 {@code ExecutionRequestRefContext}。 + * @return 节点执行的上下文信息类 + */ + default ExecutionRequestRefContext getExecutionRequestRefContext() { + return (ExecutionRequestRefContext) getParameter("executionRequestRefContext"); + } + + default R setExecutionRequestRefContext(ExecutionRequestRefContext executionRequestRefContext) { + setParameter("executionRequestRefContext", executionRequestRefContext); + return (R) this; + } + + /** + * DSS 工作流设置的所有全局变量 + * 由于第三方 AppConn 的 Job 也可能支持添加自定义变量,我们建议自定义变量的作用域如下: + * 1. 如果 用户同时在 DSS 工作流和 第三方 AppConn 的 Job 都设置了同一个自定义变量,则以第三方 AppConn 的 Job 的为准 + * 2. 否则使用 DSS 工作流的自定义变量 + * @return 返回 DSS 工作流所设置的所有全局变量 + */ + default Map getVariables() { + return (Map) getParameter("variables"); + } + + default R setVariables(Map variables) { + setParameter("variables", variables); + return (R) this; + } + + /** + * 这是一个非常重要的属性,当第三方 AppConn 支持按照选定的日期进行跑批时,那么在执行对应的 refJob 时,请一定要带上该 + * 参数到第三方 refJob,以便一些调度系统想要做历史跑批时,可以正确地执行其对应日期的作业。 + * @return null if not exists, otherwise return the date string, the date format is yyyyMMdd. + */ + default String getRunDate() { + if(getVariables().containsKey("run_date")) { + return (String) getVariables().get("run_date"); + } else { + return null; + } + } + + /** + * If the third-part AppConn has no project, and don't wants to integrate with DSS context service, + * please use this class to asyncly execute refJob. + */ + class RefExecutionRequestRefImpl extends DevelopmentRequestRefImpl + implements RefExecutionRequestRef {} + + /** + * If the third-part AppConn has project, and don't wants to integrate with DSS context service, + * please use this class to asyncly execute refJob. + */ + class RefExecutionProjectRequestRef extends DevelopmentRequestRefImpl + implements RefExecutionRequestRef, + ProjectRefRequestRef {} + + /** + * If the third-part AppConn wants to integrate with DSS context service, + * please use this class to asyncly execute refJob. + */ + class RefExecutionContextRequestRef extends DevelopmentRequestRefImpl + implements RefExecutionRequestRef, + DSSContextRequestRef {} + + /** + * If the third-part AppConn has project, and wants to integrate with DSS context service, + * please use this class to asyncly execute refJob. + */ + class RefExecutionProjectWithContextRequestRef extends DevelopmentRequestRefImpl + implements RefExecutionRequestRef, + ProjectRefRequestRef, + DSSContextRequestRef {} + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/AsyncRefExecutionSchedulerListener.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/AsyncRefExecutionSchedulerListener.java new file mode 100644 index 000000000..017db72f6 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/AsyncRefExecutionSchedulerListener.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.scheduler; + +import org.apache.linkis.common.listener.Event; +import org.apache.linkis.common.listener.EventListener; + + +public interface AsyncRefExecutionSchedulerListener extends EventListener { + + void onEvent(AsyncResponseRefEvent event); + + void onEventError(AsyncResponseRefEvent event, Throwable t); + + @Override + default void onEventError(Event event, Throwable t) { + if(event instanceof AsyncResponseRefEvent) { + onEventError((AsyncResponseRefEvent) event, t); + } + } +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/AsyncResponseRefEvent.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/AsyncResponseRefEvent.java new file mode 100644 index 000000000..511bc3d78 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/AsyncResponseRefEvent.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.scheduler; + +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.AsyncExecutionResponseRef; +import org.apache.linkis.common.listener.Event; + + +public class AsyncResponseRefEvent implements Event { + + private AsyncExecutionResponseRef response; + private long lastAskTime = 0; + + public AsyncResponseRefEvent(AsyncExecutionResponseRef response) { + this.response = response; + } + + public AsyncExecutionResponseRef getResponse() { + return response; + } + + public void setResponse(AsyncExecutionResponseRef response) { + this.response = response; + } + + public long getLastAskTime() { + return lastAskTime; + } + + public void setLastAskTime() { + this.lastAskTime = System.currentTimeMillis(); + } +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/LongTermRefExecutionScheduler.java b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/LongTermRefExecutionScheduler.java new file mode 100644 index 000000000..45afe0d57 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/java/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/LongTermRefExecutionScheduler.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.scheduler; + +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.AsyncExecutionResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.listener.common.LongTermRefExecutionAction; +import com.webank.wedatasphere.dss.standard.common.entity.ref.AsyncResponseRef; + + +public interface LongTermRefExecutionScheduler { + + void addAsyncResponse(AsyncExecutionResponseRef response); + + void removeAsyncResponse(LongTermRefExecutionAction action); + + AsyncResponseRef getAsyncResponse(LongTermRefExecutionAction action); + + void start(); + + void stop(); + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/scala/com/webank/wedatasphere/dss/standard/app/development/listener/conf/RefExecutionConfiguration.scala b/dss-standard/development-standard/development-process-standard-execution/src/main/scala/com/webank/wedatasphere/dss/standard/app/development/listener/conf/RefExecutionConfiguration.scala new file mode 100644 index 000000000..9bb74e532 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/scala/com/webank/wedatasphere/dss/standard/app/development/listener/conf/RefExecutionConfiguration.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.conf + +import org.apache.linkis.common.conf.{CommonVars, TimeType} + + +object RefExecutionConfiguration { + + val JOB_HISTORY_APPLICATION_NAME = CommonVars("wds.dss.appconn.jobhistory.service.name", "linkis-ps-publicservice") + val CALL_BACK_URL = CommonVars("wds.dss.appconn.ref-execution.callback.url", "/api/rest_j/v1/engineconn/callback") + + val ASYNC_REF_EXECUTION_SCHEDULER_QUEUE_SIZE = CommonVars("wds.dss.appconn.ref-execution.scheduler.queue.size", 2000) + val ASYNC_REF_EXECUTION_SCHEDULER_THREAD_SIZE = CommonVars("wds.dss.appconn.ref-execution.scheduler.thread.max", 20) + + val CALLBACK_REF_EXECUTION_REFRESH_INTERVAL = CommonVars("wds.dss.appconn.ref-execution.callback.refresh.interval", new TimeType("2m")) + +} diff --git a/dss-standard/development-standard/development-process-standard-execution/src/main/scala/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/ListenerEventBusRefExecutionScheduler.scala b/dss-standard/development-standard/development-process-standard-execution/src/main/scala/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/ListenerEventBusRefExecutionScheduler.scala new file mode 100644 index 000000000..abe0036ca --- /dev/null +++ b/dss-standard/development-standard/development-process-standard-execution/src/main/scala/com/webank/wedatasphere/dss/standard/app/development/listener/scheduler/ListenerEventBusRefExecutionScheduler.scala @@ -0,0 +1,121 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.listener.scheduler + +import java.util.concurrent.ArrayBlockingQueue + +import com.webank.wedatasphere.dss.standard.app.development.listener.common._ +import com.webank.wedatasphere.dss.standard.app.development.listener.conf.RefExecutionConfiguration._ +import com.webank.wedatasphere.dss.standard.app.development.listener.exception.AppConnExecutionErrorException +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef.ExecutionResponseRefBuilder +import com.webank.wedatasphere.dss.standard.app.development.listener.ref.{AsyncExecutionResponseRef, ExecutionResponseRef} +import org.apache.commons.lang.time.DateFormatUtils +import org.apache.linkis.common.listener.ListenerEventBus +import org.apache.linkis.common.utils.{ByteTimeUtils, Utils} + +import scala.collection.JavaConverters._ + + +class ListenerEventBusRefExecutionScheduler(eventQueueCapacity: Int, name: String)(listenerConsumerThreadSize: Int) + extends LongTermRefExecutionScheduler { + + private val listenerEventBus = new ListenerEventBus[AsyncRefExecutionSchedulerListener, AsyncResponseRefEvent](eventQueueCapacity, name)(listenerConsumerThreadSize) { + override protected val dropEvent: DropEvent = new DropEvent { + override def onDropEvent(event: AsyncResponseRefEvent): Unit = throw new AppConnExecutionErrorException(95536, "LongTermNodeExecutionScheduler is full, please ask admin for help!") + override def onBusStopped(event: AsyncResponseRefEvent): Unit = throw new AppConnExecutionErrorException(95536, "LongTermNodeExecutionScheduler is stopped, please ask admin for help!") + } + override def doPostEvent(listener: AsyncRefExecutionSchedulerListener, event: AsyncResponseRefEvent): Unit = { + listener.onEvent(event) + } + } + + def this() = { + this(ASYNC_REF_EXECUTION_SCHEDULER_QUEUE_SIZE.getValue, "Async-NodeExecution-Scheduler")(ASYNC_REF_EXECUTION_SCHEDULER_THREAD_SIZE.getValue) + getAsyncRefExecutionSchedulerListeners.foreach(listenerEventBus.addListener) + } + + private val eventQueue = { + val ru = scala.reflect.runtime.universe + val classMirror = ru.runtimeMirror(getClass.getClassLoader) + val listenerEventBusClass = classMirror.reflect(listenerEventBus) + val field1 = ru.typeOf[ListenerEventBus[_, _]].decl(ru.TermName("eventQueue")).asMethod + val result = listenerEventBusClass.reflectMethod(field1) + result() match { + case queue: ArrayBlockingQueue[AsyncResponseRefEvent] => queue + } + } + + protected def getAsyncRefExecutionSchedulerListeners: Array[AsyncRefExecutionSchedulerListener] = { + Array(new AsyncRefExecutionSchedulerListener() { + override def onEvent(event: AsyncResponseRefEvent): Unit = if(!event.getResponse.isCompleted) { + val response = event.getResponse + val action = response.getAction.asInstanceOf[AbstractRefExecutionAction] + if(action.isKilledFlag){ + val resultResponse = response.getRefExecutionOperation.result(response.getAction) + onEventCompleted(event, resultResponse) + } + if(response.getMaxLoopTime > 0 && System.currentTimeMillis - response.getStartTime >= response.getMaxLoopTime) { + onEventError(event, new AppConnExecutionErrorException(75533, s"AppConnNode Execution is overtime! StartTime is ${DateFormatUtils.format(response.getStartTime, "yyyy-MM-dd HH:mm:ss")}, maxWaitTime is " + + ByteTimeUtils.msDurationToString(response.getMaxLoopTime))) + return + } + val period = System.currentTimeMillis() - event.getLastAskTime + if(period < response.getAskStatePeriod) { + if(period < 10) Utils.sleepQuietly(100) + if(!response.isCompleted) addEvent(event) + return + } + val state = response.getRefExecutionOperation.state(response.getAction) + if(state.isCompleted) { + val resultResponse = response.getRefExecutionOperation.result(response.getAction) + onEventCompleted(event, resultResponse) + } else if(!response.isCompleted) { + event.setLastAskTime() + addEvent(event) + } + } + + private def onEventCompleted(event: AsyncResponseRefEvent, response: ExecutionResponseRef): Unit = { + event.getResponse.setCompleted(response) + } + + override def onEventError(event: AsyncResponseRefEvent, t: Throwable): Unit = t match { + case e: Exception => + val responseRef = new ExecutionResponseRefBuilder().setException(e).error() + onEventCompleted(event, responseRef) + } + }) + } + + override def addAsyncResponse(response: AsyncExecutionResponseRef): Unit = + addEvent(new AsyncResponseRefEvent(response)) + + protected def addEvent(event: AsyncResponseRefEvent): Unit = synchronized { + listenerEventBus.post(event) + } + + override def removeAsyncResponse(action: LongTermRefExecutionAction): Unit = { + + } + + override def getAsyncResponse(action: LongTermRefExecutionAction): AsyncExecutionResponseRef = + eventQueue.iterator().asScala.find(_.getResponse.getAction == action).map(_.getResponse).orNull + + override def start(): Unit = listenerEventBus.start() + + override def stop(): Unit = listenerEventBus.stop() +} \ No newline at end of file diff --git a/dss-standard/development-standard/development-process-standard/pom.xml b/dss-standard/development-standard/development-process-standard/pom.xml new file mode 100644 index 000000000..d1c61661c --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/pom.xml @@ -0,0 +1,63 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-development-process-standard + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + \ No newline at end of file diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/AbstractDevelopmentOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/AbstractDevelopmentOperation.java new file mode 100644 index 000000000..921240736 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/AbstractDevelopmentOperation.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.DevelopmentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.sso.operation.AbstractOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public abstract class AbstractDevelopmentOperation, V extends ResponseRef> + extends AbstractOperation implements DevelopmentOperation { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + /** + * Override this method and return null by default. + * The reason of returning null is that, it is possible that many third-part AppConn may not use SSORequestOperation + * which dependents the HttpClient of Linkis + * to request the third-part system, so override in this method to avoid to implement it in these subclasses. + *
+ * Notice: if you want to use SSORequestOperation, please override this method. + * @return null by default. + */ + @Override + protected String getAppConnName() { + return null; + } + + @Override + public final void setDevelopmentService(DevelopmentService service) { + this.service = service; + } +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/DevelopmentOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/DevelopmentOperation.java new file mode 100644 index 000000000..11a0b2991 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/DevelopmentOperation.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + + +import com.webank.wedatasphere.dss.standard.app.development.ref.DevelopmentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.service.Operation; + +public interface DevelopmentOperation, V extends ResponseRef> extends Operation { + + void setDevelopmentService(DevelopmentService service); + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefCopyOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefCopyOperation.java new file mode 100644 index 000000000..c23bd2300 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefCopyOperation.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.CopyRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public interface RefCopyOperation> + extends DevelopmentOperation { + + /** + * 该方法会尝试请求第三方应用工具,在 refProject 下将某个第三方 AppConn 的 refJob 复制为一个全新的 refJob, + * 关于 requestRef 的定义,请参考 {@code CopyRequestRef}。
+ * 请返回一个在该 refProject 下的全新 refJob。由于该方法一般在工作流新增版本或是复制工作流时使用,所以为了 + * 保持相关节点版本的连续性和可识别性,强烈建议根据 requestRef 提供的 newVersion,为 refJob 命名。 + * 例如:
+ * requestRef 的 name 为 test,newVersion 为 v2,建议第三方 AppConn 在复制时,新的 refJob 命名为 test_v2。 + * 如果更新成功,请返回一个带有成功标识的 {@code ResponseRef},例如:ResponseRef.newExternalBuilder().success(); + * 否则请返回一个带有错误信息的 ResponseRef,例如:ResponseRef.newExternalBuilder().error("error msg.") + * 请注意:requestRef 会带上最新的 DSSJob 的所有信息,而不是本次需要更新的信息,各第三方节点可按需更新相关内容。 + * @param requestRef 带上最新的 DSSJob 的所有信息的 requestRef + * @return 包含了成功或失败标识的 ResponseRef + * @throws ExternalOperationFailedException 复制失败时抛出该异常 + */ + RefJobContentResponseRef copyRef(K requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefCreationOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefCreationOperation.java new file mode 100644 index 000000000..f6b003377 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefCreationOperation.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.DSSJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public interface RefCreationOperation> + extends DevelopmentOperation { + + /** + * 该方法会尝试请求第三方应用工具,在 refProject 下创建一个与 DSS Job 一对一关联的第三方 refJob。 + * requestRef 的具体内容,请参考:{@code DSSJobContentRequestRef}。
+ * 如果创建成功,请返回一个包含了第三方应用工具的 job ID(命名为 refJobId)的 {@code RefJobContentResponseRef}, + * 以便 DSS 接下来可以使用 refJobId 来管理第三方应用工具对应 refProject 工程下的 refJob。 + *
+ * 返回的 refJobId 是其他 DevelopmentOperation 能够操作这个第三方应用工具的 Job 的基础。DSS 在调用其他 + * DevelopmentOperation 时,会将该 refJobId 作为方法参数传入,以便用户能正常找到对应的 refJob 进行相应操作。 + * @param requestRef 包含了 DSSJob 信息的 requestRef + * @return 包含了第三方应用工具的 job ID(命名为 refJobId)的 RefJobContentResponseRef + * @throws ExternalOperationFailedException 创建失败时抛出该异常 + */ + RefJobContentResponseRef createRef(K requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefDeletionOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefDeletionOperation.java new file mode 100644 index 000000000..c358325e0 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefDeletionOperation.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public interface RefDeletionOperation> extends DevelopmentOperation { + + /** + * 该方法会尝试请求第三方应用工具,在 refProject 下删除第三方 AppConn 的 refJob。 + * 如果删除成功,请返回一个带有成功标识的 {@code ResponseRef},例如:ResponseRef.newExternalBuilder().success(); + * 否则请返回一个带有错误信息的 ResponseRef,例如:ResponseRef.newExternalBuilder().error("error msg.") + * @param requestRef 包含了第三方 refJob 信息的 requestRef + * @return 包含了成功或失败标识的 ResponseRef + * @throws ExternalOperationFailedException 删除失败时抛出该异常 + */ + ResponseRef deleteRef(K requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefExecutionOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefExecutionOperation.java new file mode 100644 index 000000000..28fb319dd --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefExecutionOperation.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public interface RefExecutionOperation> + extends DevelopmentOperation { + + /** + * 执行第三方 AppConn 的 refJob。
+ * requestRef 一般为 {@code RefExecutionRequestRef},该接口定义了四个实现类,您可以根据需要去使用其中的一个 RefExecutionRequestRef 实现类。 + * {@code RefExecutionRequestRef} 中的 {@code ExecutionRequestRefContext} 为该节点执行的上下文信息类,包含了该第三方节点执行所需的 + * 所有信息,以及如果该第三方节点想要将执行完成后的结果集写入到 Linkis 中,以供下游节点使用时,{@code ExecutionRequestRefContext} 也提供了 + * 创建各种 createResultSetWriter 的方法;或是如果该第三方节点想要获取上游节点的结果集文件内容时,{@code ExecutionRequestRefContext} 也提供了获取 + * LinkisJob 信息和相关结果集文件路径的方法,可基于结果集路径调用 {@code ExecutionRequestRefContext.getResultSetReader()} 获取结果集文件内容。 + * 更多关于执行上下文的信息,请参考:{@code ExecutionRequestRefContext}。 + *
+ * 通常情况下,如果您的这个 refJob 是一个运行时间较长的任务,如执行时间会超过2分钟,那么走异步提交执行的方式无疑对提升系统的高并发能力有非常大的帮助。 + * 这时我们建议大家继承 RefExecutionOperation 的子类 {@code LongTermRefExecutionOperation} 来实现异步提交执行能力。 + * 当绝大部分的 refJob 的执行时间会超过10分钟,我们强烈要求必须实现 {@code LongTermRefExecutionOperation},否则当调度作业越来越多时(1小时超过1万个), + * 系统可能会出现高并发场景下的性能瓶颈。 + * 更多关于常驻型作业执行的信息,请参考:{@code LongTermRefExecutionOperation}。 + *
+ * 如果您的 refJob 作业是短时间运行的作业,您可以直接实现该接口提供执行能力。当您直接实现该接口时,该方法要求返回一个执行结果。 + * 如果您只想返回执行成功,可返回 {@code RequestRef.newExternalBuilder.success()};如果执行失败,可返回一个包含了 exception 的 {@code ExecutionResponseRef} + * @param requestRef 包含了第三方 refJob 信息的 requestRef + * @return 返回一个执行结果,如果您只想返回执行成功,可返回 {@code RequestRef.newExternalBuilder.success()};执行失败可返回一个包含了 exception 的 {@code ExecutionResponseRef} + * @throws ExternalOperationFailedException 执行发生不可预知的错误时,抛出该异常 + */ + ResponseRef execute(K requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefExportOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefExportOperation.java new file mode 100644 index 000000000..9a084bd51 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefExportOperation.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.ExportResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * 支持将第三方 AppConn 的 Job 导出成 Linkis BML 物料或 InputStream 字节流。 + * @param + */ +public interface RefExportOperation> extends DevelopmentOperation { + + /** + * Now, DSS only supports to export a third-part AppConn job to Linkis BML resources or {@code InputStream}. + *
+ * So, if third-part AppConn want to achieve the {@code RefExportOperation}, it is necessary that the third-part AppConn + * must upload the meta and resources of third-part AppConn job to Linkis BML at first, or packages the the meta and + * resources of third-part AppConn job as a InputStream, then return the resourceMap to DSS. + * For more detail, please see {@code ExportResponseRef} or {@code ImportRequestRef}. + */ + ExportResponseRef exportRef(K requestRef) throws ExternalOperationFailedException; + +} + + diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefImportOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefImportOperation.java new file mode 100644 index 000000000..17ae8d376 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefImportOperation.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * 通过传入 Linkis BML 物料或 InputStream 字节流,第三方 AppConn 需支持将其转换成一个 refJob。 + * @param + */ +public interface RefImportOperation> + extends DevelopmentOperation { + + /** + * The resourceMap in {@code ImportRequestRef} is the content of {@code ExportRequestRef} exported. + *
+ * Now, DSS only supports to import Linkis BML resources or {@code InputStream}, + * so if you want to choose Linkis BML resources to ex/import, the resourceMap is consisted of `resourceId` + * and `version`, otherwise the resourceMap is consisted of inputStream. + *
+ * For more information about {@code requestRef}, please see {@code ImportRequestRef}. + * @return a refJobContent related to a only third appConn refJob which created by the resourceMap. + */ + RefJobContentResponseRef importRef(K requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefQueryJumpUrlOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefQueryJumpUrlOperation.java new file mode 100644 index 000000000..b209d206d --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefQueryJumpUrlOperation.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.QueryJumpUrlRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.QueryJumpUrlResponseRef; + +/** + * + * 关于 QueryJumpUrlRequestRef,可参考 {@code QueryJumpUrlRequestRef}。 + * 返回的 {@code ResponseRef} 为:{@code QueryJumpUrlResponseRef},用于当用户在前端双击该工作流节点时,返回一个可以跳转的 jumpURL; + * 否则请返回一个带有错误信息的 ResponseRef,例如:ResponseRef.newExternalBuilder().error("error msg.") + * + * @author enjoyyin + * @date 2022-04-14 + * @since 1.1.0 + */ +public interface RefQueryJumpUrlOperation, + V extends QueryJumpUrlResponseRef> extends RefQueryOperation { +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefQueryOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefQueryOperation.java new file mode 100644 index 000000000..a85c13016 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefQueryOperation.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + + +public interface RefQueryOperation, V extends ResponseRef> + extends DevelopmentOperation { + + /** + * 该方法会尝试请求第三方应用工具,在 refProject 下获取第三方 AppConn 的 refJob 的一些信息。 + * DSS 框架允许用户实现多个 RefQueryOperation,用于查询 refJob 的个性化信息。 + * 目前 DSS 框架层最希望第三方 AppConn 实现的是 {@code RefQueryJumpUrlOperation}。 + * @param requestRef 带上最新的 DSSJob 的所有信息的 requestRef + * @return 带有第三方应用 refJob 希望返回的信息。 + * @throws ExternalOperationFailedException 查询失败时抛出该异常 + */ + V query(K requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefUpdateOperation.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefUpdateOperation.java new file mode 100644 index 000000000..059c73080 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/operation/RefUpdateOperation.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.operation; + +import com.webank.wedatasphere.dss.standard.app.development.ref.UpdateRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public interface RefUpdateOperation> + extends DevelopmentOperation { + + /** + * 该方法会尝试请求第三方应用工具,在 refProject 下更新第三方 AppConn 的 refJob。 + * 如果更新成功,请返回一个带有成功标识的 {@code ResponseRef{@code ,例如:ResponseRef.newExternalBuilder().success(); + * 否则请返回一个带有错误信息的 {@code ResponseRef},例如:ResponseRef.newExternalBuilder().error("error msg.") + * 请注意:requestRef 会带上最新的 DSSJob 的所有信息,而不是本次需要更新的信息,各第三方节点可按需更新相关内容。 + * @param requestRef 带上最新的 DSSJob 的所有信息的 requestRef + * @return 包含了成功或失败标识的 ResponseRef + * @throws ExternalOperationFailedException 更新失败时抛出该异常 + */ + ResponseRef updateRef(K requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/CopyRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/CopyRequestRef.java new file mode 100644 index 000000000..96c3b590d --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/CopyRequestRef.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.ref; + + +public interface CopyRequestRef> + extends RefJobContentRequestRef { + + default R setNewVersion(String version) { + setParameter("newVersion", version); + return (R) this; + } + + /** + * The new version comes from DSS Orchestrator framework. + * When the orchestrator, such as DSSWorkflow, added a new version, we hope all nodes of this workflow, + * can also update a new version. + * @return the new version of the orchestrator, such as DSSWorkflow. + */ + default String getNewVersion() { + return (String) getParameter("newVersion"); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DSSContextRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DSSContextRequestRef.java new file mode 100644 index 000000000..aab6d9ecb --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DSSContextRequestRef.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +/** + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public interface DSSContextRequestRef> extends DevelopmentRequestRef { + + default String getContextId() { + return (String) getParameter("dssContextId"); + } + + default R setContextId(String contextIdStr) { + setParameter("dssContextId", contextIdStr); + return (R) this; + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DSSJobContentRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DSSJobContentRequestRef.java new file mode 100644 index 000000000..08c10bb79 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DSSJobContentRequestRef.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public interface DSSJobContentRequestRef> + extends DevelopmentRequestRef { + + /** + * When some DevelopmentOperations, such as {@code RefCreationOperation}, {@code RefImportOperation} are called, the dssJobContent will be set in, so we can use it to + * create a one-to-one only related third appConn refJob. + *
+ * {@code DSSJobContentConstant} contains all the keys of {@code dssJobContent}, please see {@code DSSJobContentConstant} to know all these keys + * DSS will fill in. + *
+ * This method is usually called by DSS framework, the third-part AppConn just need to use {@code getDSSJobContent()} to + * get the dssJobContent to create a one-to-one only related third appConn refJob. + * @param dssJobContent a dssJobContent created by users in DSS front-web. + * @return this class + */ + default R setDSSJobContent(Map dssJobContent) { + setParameter("dssJobContent", dssJobContent); + return (R) this; + } + + /** + * This is the content of the DSS job content, when users try to create a DSS job in DSS front-web. + * the third-part AppConn usually will call this method to get dssJobContent to do some operation. + * @return a dssJobContent related a dss job created by dss framework. + */ + default Map getDSSJobContent() { + return (Map) getParameter("dssJobContent"); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DevelopmentRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DevelopmentRequestRef.java new file mode 100644 index 000000000..d6597263b --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/DevelopmentRequestRef.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.ref.WorkspaceRequestRef; + +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public interface DevelopmentRequestRef> extends WorkspaceRequestRef { + + default R setUserName(String userName) { + setParameter("userName", userName); + return (R) this; + } + + default String getUserName() { + return (String) getParameter("userName"); + } + + R setName(String name); + + R setDSSLabels(List dssLabels); + + R setType(String type); + + R setWorkspace(Workspace workspace); + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ExportResponseRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ExportResponseRef.java new file mode 100644 index 000000000..40d57be9a --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ExportResponseRef.java @@ -0,0 +1,63 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public interface ExportResponseRef extends ResponseRef { + + /** + * Now, DSS only supports to import/export Linkis BML resources or one inputStream, + * so the resourceMap is consisted of `resourceId` and `version` if you choose Linkis BML resources, + * or `InputStream` and `Closeable` if you choose Stream resources. + *
+ * If third-part AppConn want to achieve the {@code RefExportOperation}, it is necessary that the third-part AppConn + * must upload the meta and resources of third-part AppConn job to Linkis BML at first, or packages the the meta and + * resources of third-part AppConn job as a InputStream, then return the resourceMap to DSS. + * @return a refJobContent related a only third appConn refJob + */ + Map getResourceMap(); + + default boolean isLinkisBMLResources() { + return getResourceMap().containsKey(ImportRequestRef.RESOURCE_ID_KEY) + && getResourceMap().containsKey(ImportRequestRef.RESOURCE_VERSION_KEY); + } + + static ExportResponseRefBuilder newBuilder() { + return new ExportResponseRefBuilder(); + } + + class ExportResponseRefBuilder + extends ResponseRefBuilder.ExternalResponseRefBuilder { + + private Map resourceMap; + + public ExportResponseRefBuilder setResourceMap(Map resourceMap) { + this.resourceMap = resourceMap; + return this; + } + + class ExportResponseRefImpl extends ResponseRefImpl implements ExportResponseRef { + public ExportResponseRefImpl() { + super(ExportResponseRefBuilder.this.responseBody, ExportResponseRefBuilder.this.status, + ExportResponseRefBuilder.this.errorMsg, ExportResponseRefBuilder.this.responseMap); + } + @Override + public Map getResourceMap() { + return resourceMap; + } + } + + @Override + protected ExportResponseRefImpl createResponseRef() { + return new ExportResponseRefImpl(); + } + } +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ImportRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ImportRequestRef.java new file mode 100644 index 000000000..f2c1b464d --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ImportRequestRef.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.ref; + + +import java.util.Map; + +public interface ImportRequestRef> + extends CopyRequestRef { + + default R setResourceMap(Map resourceMap) { + setParameter("resourceMap", resourceMap); + return (R) this; + } + + /** + * ResourceMap is the content of ExportRequestRef exported. + *
+ * Now, DSS only supports to import BML resources or one inputStream, + * so when you want to use BML resources to import/export, the resourceMap must be + * consisted of `resourceId` and `version`; and if you want to use stream to + * import/export, the resourceMap must be consisted of `inputStream` and `closeable`. + *
+ * The `closeable` is remained field that if your inputStream comes from HttpClient, + * you can set the `closeable` to CloseableHttpResponse to void exception. + * @return the content of ExportRequestRef exported. + */ + default Map getResourceMap() { + return (Map) getParameter("resourceMap"); + } + + default boolean isLinkisBMLResources() { + return true; + } + + default boolean isStreamResources() { + return !isLinkisBMLResources(); + } + + String RESOURCE_ID_KEY = "resourceId"; + String RESOURCE_VERSION_KEY = "version"; + + String INPUT_STREAM_KEY = "inputStream"; + String CLOSEABLE_KEY = "closeable"; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ProjectRefRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ProjectRefRequestRef.java new file mode 100644 index 000000000..1086338a8 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/ProjectRefRequestRef.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +/** + * @author enjoyyin + * @date 2022-03-09 + * @since 1.0.0 + */ +public interface ProjectRefRequestRef> extends DevelopmentRequestRef { + + default R setRefProjectId(Long refProjectId) { + this.setParameter("refProjectId", refProjectId); + return (R) this; + } + + /** + * 返回的是第三方 AppConn 的工程 Id,由第三方 AppConn {@code ProjectCreationOperation} 的实现类返回的 refProjectId。 + * @return 返回的是第三方 AppConn 的工程Id + */ + default Long getRefProjectId() { + return (Long) this.getParameter("refProjectId"); + } + + default R setDSSProjectId(Long dssProjectId) { + this.setParameter("dssProjectId", dssProjectId); + return (R) this; + } + + /** + * 返回的是 DSSProject 的工程 Id + * @return 返回的是 DSSProject 的工程 Id + */ + default Long getDSSProjectId() { + return (Long) this.getParameter("dssProjectId"); + } + + default R setProjectName(String projectName) { + this.setParameter("projectName", projectName); + return (R) this; + } + + /** + * 返回的是 DSSProject 的工程名 + * @return 返回的是 DSSProject 的工程名 + */ + default String getProjectName() { + return (String) this.getParameter("projectName"); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/QueryJumpUrlRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/QueryJumpUrlRequestRef.java new file mode 100644 index 000000000..efdd93253 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/QueryJumpUrlRequestRef.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; + +/** + * @author enjoyyin + * @date 2022-04-14 + * @since 1.1.0 + */ +public interface QueryJumpUrlRequestRef> extends RefJobContentRequestRef { + + default R setSSOUrlBuilderOperation(SSOUrlBuilderOperation ssoUrlBuilderOperation) { + setParameter("ssoUrlBuilderOperation", ssoUrlBuilderOperation); + return (R) this; + } + + /** + * SSOUrlBuilderOperation 用于封装出第三方系统 refJob 对应的前端页面的 URL。 + * 该 SSOUrlBuilderOperation 已填充了 setWorkspace setDSSUrl setAppName addCookie 方法,用户需按实际需要,填充 + * setReqUrl、redirectTo,以及 addQueryParameter。 + *

+ * 具体解释如下: + * setReqUrl: 设置第三方应用的请求 URL,该 URL 必须要能进入到第三方系统后台的 Filter,详见DSS第一级规范 + * redirectTo:第三方系统的前端页面 URL。 + * addQueryParameter:额外的参数,DSS 支持自动替换的参数详见 {@code QueryJumpUrlConstant} + * @return SSOUrlBuilderOperation + */ + default SSOUrlBuilderOperation getSSOUrlBuilderOperation() { + return (SSOUrlBuilderOperation) getParameter("ssoUrlBuilderOperation"); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/QueryJumpUrlResponseRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/QueryJumpUrlResponseRef.java new file mode 100644 index 000000000..d003e1ab3 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/QueryJumpUrlResponseRef.java @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +/** + * @author enjoyyin + * @date 2022-03-11 + * @since 0.5.0 + */ +public interface QueryJumpUrlResponseRef extends ResponseRef { + + /** + * When user wants to create a new workflow node of third-part AppConn in front-web. + * After creating, user tries to open this workflow node in front-web, in this time, + * front-web will request dss framework to get the jump url, so dss framework will + * try to get jump url of the front-web page of the only third-party AppConn refJob + * by using RefQueryOperation of the third-part AppConn. + * @return the jump url related to the front-web page of the only third-party AppConn refJob + */ + String getJumpUrl(); + + static QueryJumpUrlResponseRefBuilder newBuilder() { + return new QueryJumpUrlResponseRefBuilder(); + } + + class QueryJumpUrlResponseRefBuilder + extends ResponseRefBuilder.ExternalResponseRefBuilder { + + private String jumpUrl; + + public QueryJumpUrlResponseRefBuilder setJumpUrl(String jumpUrl) { + this.jumpUrl = jumpUrl; + return this; + } + + class QueryJumpUrlResponseRefImpl extends ResponseRefImpl implements QueryJumpUrlResponseRef { + public QueryJumpUrlResponseRefImpl() { + super(QueryJumpUrlResponseRefBuilder.this.responseBody, QueryJumpUrlResponseRefBuilder.this.status, + QueryJumpUrlResponseRefBuilder.this.errorMsg, QueryJumpUrlResponseRefBuilder.this.responseMap); + } + @Override + public String getJumpUrl() { + return jumpUrl; + } + } + + @Override + protected QueryJumpUrlResponseRefImpl createResponseRef() { + return new QueryJumpUrlResponseRefImpl(); + } + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/RefJobContentRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/RefJobContentRequestRef.java new file mode 100644 index 000000000..c0e2b5af3 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/RefJobContentRequestRef.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +import java.util.Map; + +/** + * This class is used to represent a one-to-one relationship about DSS job with third appConn refJob. + * When some kinds of DevelopmentOperation are called, they usually need these refJobContent which can + * help to find the only refJob in third-appConn to operate the third-appConn. + * The refJobContent is returned from the DevelopmentOperation of RefCreationOperation, RefImportOperation, + * DSS will persistence these returned refJobContents, and set in when call some other DevelopmentOperations, + * such as RefDeletionOperation, RefUpdateOperation. + * + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public interface RefJobContentRequestRef> + extends DevelopmentRequestRef { + + /** + * When some DevelopmentOperation is called, the refJobContent will be set in, so we can use it to + * operate the relative third appConn refJob. + * This method is usually called by DSS framework, the third-part AppConn just need to use getRefJobContent to + * get the refJobContent. + * @param refJobContent a refJobContent related a only third appConn refJob + * @return this class + */ + default R setRefJobContent(Map refJobContent) { + setParameter("refJobContent", refJobContent); + return (R) this; + } + + /** + * This is the content of the third appConn returned, when the RefCreationOperation is called. + * the third-part AppConn usually will call this method to get refJobContent to do some operation. + * @return a refJobContent related a only third appConn refJob + */ + default Map getRefJobContent() { + return (Map) getParameter("refJobContent"); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/RefJobContentResponseRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/RefJobContentResponseRef.java new file mode 100644 index 000000000..dda8c0845 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/RefJobContentResponseRef.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +import java.util.Map; + +/** + * The class is used by the DevelopmentOperation of RefCreationOperation, RefImportOperation, which should return + * this class to deliver a refJobContent related a only third appConn refJob to DSS framework. + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public interface RefJobContentResponseRef extends ResponseRef { + + /** + * This refJobContent must not be empty. + * The third appConn should ensure this refJObContent can relate to a only third appConn refJob. + * @return a refJobContent related a only third appConn refJob + */ + Map getRefJobContent(); + + static RefJobContentResponseRefBuilder newBuilder() { + return new RefJobContentResponseRefBuilder(); + } + + class RefJobContentResponseRefBuilder + extends ResponseRefBuilder.ExternalResponseRefBuilder { + + private Map refJobContent; + + public RefJobContentResponseRefBuilder setRefJobContent(Map refJobContent) { + this.refJobContent = refJobContent; + return this; + } + + class RefJobContentResponseRefImpl extends ResponseRefImpl implements RefJobContentResponseRef{ + public RefJobContentResponseRefImpl() { + super(RefJobContentResponseRefBuilder.this.responseBody, RefJobContentResponseRefBuilder.this.status, + RefJobContentResponseRefBuilder.this.errorMsg, RefJobContentResponseRefBuilder.this.responseMap); + } + @Override + public Map getRefJobContent() { + return refJobContent; + } + } + + @Override + protected RefJobContentResponseRef createResponseRef() { + return new RefJobContentResponseRefImpl(); + } + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/UpdateRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/UpdateRequestRef.java new file mode 100644 index 000000000..d9cfb45ce --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/UpdateRequestRef.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.ref; + + +public interface UpdateRequestRef> extends RefJobContentRequestRef, DSSJobContentRequestRef { +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/DevelopmentRequestRefImpl.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/DevelopmentRequestRefImpl.java new file mode 100644 index 000000000..b9c489cdc --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/DevelopmentRequestRefImpl.java @@ -0,0 +1,48 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref.impl; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.development.ref.DevelopmentRequestRef; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRefImpl; + +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public class DevelopmentRequestRefImpl> extends RequestRefImpl + implements DevelopmentRequestRef { + + private Workspace workspace; + + @Override + public R setName(String name) { + this.name = name; + return (R) this; + } + + @Override + public R setDSSLabels(List dssLabels) { + this.dssLabels = dssLabels; + return (R) this; + } + + @Override + public R setType(String type) { + this.type = type; + return (R) this; + } + + @Override + public R setWorkspace(Workspace workspace) { + this.workspace = workspace; + return (R) this; + } + + @Override + public Workspace getWorkspace() { + return workspace; + } +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/OnlyDevelopmentRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/OnlyDevelopmentRequestRef.java new file mode 100644 index 000000000..538493da1 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/OnlyDevelopmentRequestRef.java @@ -0,0 +1,89 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref.impl; + +import com.webank.wedatasphere.dss.standard.app.development.ref.*; + +/** + * We can find the only difference of these classes between OnlyDevelopmentRequestRef and ThirdlyRequestRef, + * is that, OnlyDevelopmentRequestRef is used for those third-part AppConns which has no projects, and + * ThirdlyRequestRef is used for those third-part AppConns which not only contains projects but also contains + * development process. + * So, if your system has no project, please use the classes of OnlyDevelopmentRequestRef, otherwise please + * use the classes of ThirdlyRequestRef. + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public interface OnlyDevelopmentRequestRef { + + class DSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements DSSJobContentRequestRef {} + + class RefJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements RefJobContentRequestRef {} + + class DSSJobContentWithContextRequestRef extends DevelopmentRequestRefImpl + implements DSSJobContentRequestRef, DSSContextRequestRef {} + + class RefJobContentWithContextRequestRef extends DevelopmentRequestRefImpl + implements RefJobContentRequestRef, DSSContextRequestRef {} + + class CopyRequestRefImpl extends DevelopmentRequestRefImpl + implements CopyRequestRef {} + + class CopyWithDSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements CopyRequestRef, DSSJobContentRequestRef {} + + class CopyWitContextRequestRefImpl extends DevelopmentRequestRefImpl + implements CopyRequestRef, + DSSContextRequestRef {} + + class CopyWitContextAndDSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements CopyRequestRef, + DSSContextRequestRef, + DSSJobContentRequestRef{} + + class ImportRequestRefImpl extends DevelopmentRequestRefImpl + implements ImportRequestRef {} + + class ImportWithStreamRequestRefImpl extends DevelopmentRequestRefImpl + implements ImportRequestRef { + @Override + public boolean isLinkisBMLResources() { + return false; + } + } + + class ImportWitContextRequestRefImpl extends DevelopmentRequestRefImpl + implements ImportRequestRef, + DSSContextRequestRef{} + + class ImportWitContextAndStreamRequestRefImpl extends DevelopmentRequestRefImpl + implements ImportRequestRef, + DSSContextRequestRef { + @Override + public boolean isLinkisBMLResources() { + return false; + } + } + + class UpdateRequestRefImpl extends DevelopmentRequestRefImpl + implements UpdateRequestRef {} + + class UpdateWitContextRequestRefImpl extends DevelopmentRequestRefImpl + implements UpdateRequestRef, DSSContextRequestRef{} + + class QueryJumpUrlRequestRefImpl extends DevelopmentRequestRefImpl + implements QueryJumpUrlRequestRef {} + + class QueryJumpUrlWithContextRequestRefImpl extends DevelopmentRequestRefImpl + implements QueryJumpUrlRequestRef, + DSSContextRequestRef {} + + class QueryJumpUrlWithDSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements QueryJumpUrlRequestRef, DSSJobContentRequestRef {} + + class QueryJumpUrlWithContextAndDSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements QueryJumpUrlRequestRef, + DSSContextRequestRef, + DSSJobContentRequestRef{} +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/ThirdlyRequestRef.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/ThirdlyRequestRef.java new file mode 100644 index 000000000..2a405b1d4 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/ref/impl/ThirdlyRequestRef.java @@ -0,0 +1,95 @@ +package com.webank.wedatasphere.dss.standard.app.development.ref.impl; + +import com.webank.wedatasphere.dss.standard.app.development.ref.*; + +/** + * We can find the only difference of these classes between OnlyDevelopmentRequestRef and ThirdlyRequestRef, + * is that, OnlyDevelopmentRequestRef is used for those third-part AppConns which has no projects, and + * ThirdlyRequestRef is used for those third-part AppConns which not only contains projects but also contains + * development process. + *
+ * So, if your system has no project, please use the classes of OnlyDevelopmentRequestRef, otherwise please + * use the classes of ThirdlyRequestRef. + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public interface ThirdlyRequestRef { + + class DSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements DSSJobContentRequestRef, + ProjectRefRequestRef {} + + class DSSJobContentWithContextRequestRef extends DevelopmentRequestRefImpl + implements DSSJobContentRequestRef, DSSContextRequestRef, + ProjectRefRequestRef {} + + class RefJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements RefJobContentRequestRef, ProjectRefRequestRef {} + + class RefJobContentWithContextRequestRef extends DevelopmentRequestRefImpl + implements RefJobContentRequestRef, DSSContextRequestRef, + ProjectRefRequestRef {} + + class CopyRequestRefImpl extends DevelopmentRequestRefImpl + implements CopyRequestRef, ProjectRefRequestRef {} + + class CopyWithDSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements CopyRequestRef, ProjectRefRequestRef, + DSSJobContentRequestRef {} + + class CopyWitContextRequestRefImpl extends DevelopmentRequestRefImpl + implements CopyRequestRef, ProjectRefRequestRef, + DSSContextRequestRef {} + + class CopyWitContextAndDSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements CopyRequestRef, + DSSContextRequestRef, ProjectRefRequestRef, + DSSJobContentRequestRef{} + + class ImportRequestRefImpl extends DevelopmentRequestRefImpl + implements ImportRequestRef, ProjectRefRequestRef {} + + class ImportWithStreamRequestRefImpl extends DevelopmentRequestRefImpl + implements ImportRequestRef, ProjectRefRequestRef { + @Override + public boolean isLinkisBMLResources() { + return false; + } + } + + class ImportWitContextRequestRefImpl extends DevelopmentRequestRefImpl + implements ImportRequestRef, ProjectRefRequestRef, + DSSContextRequestRef{} + + class ImportWitContextAndStreamRequestRefImpl extends DevelopmentRequestRefImpl + implements ImportRequestRef, ProjectRefRequestRef, + DSSContextRequestRef{ + @Override + public boolean isLinkisBMLResources() { + return false; + } + } + + class UpdateRequestRefImpl extends DevelopmentRequestRefImpl + implements UpdateRequestRef, ProjectRefRequestRef {} + + class UpdateWitContextRequestRefImpl extends DevelopmentRequestRefImpl + implements UpdateRequestRef, ProjectRefRequestRef, + DSSContextRequestRef{} + + class QueryJumpUrlRequestRefImpl extends DevelopmentRequestRefImpl + implements QueryJumpUrlRequestRef, ProjectRefRequestRef {} + + class QueryJumpUrlWithContextRequestRefImpl extends DevelopmentRequestRefImpl + implements QueryJumpUrlRequestRef, ProjectRefRequestRef, + DSSContextRequestRef {} + + class QueryJumpUrlWithDSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements QueryJumpUrlRequestRef, ProjectRefRequestRef, + DSSJobContentRequestRef{} + + class QueryJumpUrlWithContextAndDSSJobContentRequestRefImpl extends DevelopmentRequestRefImpl + implements QueryJumpUrlRequestRef, ProjectRefRequestRef, + DSSContextRequestRef, DSSJobContentRequestRef {} +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractDevelopmentService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractDevelopmentService.java new file mode 100644 index 000000000..f4c577f19 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractDevelopmentService.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.development.operation.DevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppSingletonIntegrationServiceImpl; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractDevelopmentService extends AppSingletonIntegrationServiceImpl implements DevelopmentService { + + private List dssLabels = new ArrayList<>(); + private DevelopmentIntegrationStandard appStandard; + + public void setLabels(List labels) { + dssLabels = labels; + } + + @Override + public void setAppStandard(DevelopmentIntegrationStandard appStandard) { + this.appStandard = appStandard; + } + + @Override + public DevelopmentIntegrationStandard getAppStandard() { + return appStandard; + } + + @Override + public List getLabels() { + return dssLabels; + } + + @Override + protected void initOperation(DevelopmentOperation operation) { + operation.setDevelopmentService(this); + } +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefCRUDService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefCRUDService.java new file mode 100644 index 000000000..e630cb7c9 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefCRUDService.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCopyOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCreationOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.CopyRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.DSSJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.UpdateRequestRef; + +public abstract class AbstractRefCRUDService extends AbstractDevelopmentService implements RefCRUDService { + + protected abstract > RefCreationOperation createRefCreationOperation(); + + protected abstract > RefCopyOperation createRefCopyOperation(); + + protected abstract > RefUpdateOperation createRefUpdateOperation(); + + protected abstract > RefDeletionOperation createRefDeletionOperation(); + + @Override + public > RefCreationOperation getRefCreationOperation() { + return getOrCreate(this::createRefCreationOperation, RefCreationOperation.class); + } + + @Override + public > RefCopyOperation getRefCopyOperation() { + return getOrCreate(this::createRefCopyOperation, RefCopyOperation.class); + } + + @Override + public > RefUpdateOperation getRefUpdateOperation() { + return getOrCreate(this::createRefUpdateOperation, RefUpdateOperation.class); + } + + @Override + public > RefDeletionOperation getRefDeletionOperation() { + return getOrCreate(this::createRefDeletionOperation, RefDeletionOperation.class); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefExecutionService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefExecutionService.java new file mode 100644 index 000000000..05cd6d336 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefExecutionService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; + +public abstract class AbstractRefExecutionService extends AbstractDevelopmentService implements RefExecutionService { + + protected abstract RefExecutionOperation createRefExecutionOperation(); + + @Override + public RefExecutionOperation getRefExecutionOperation() { + return getOrCreate(this::createRefExecutionOperation, RefExecutionOperation.class); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefExportService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefExportService.java new file mode 100644 index 000000000..3e9c23bb7 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefExportService.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; + +public abstract class AbstractRefExportService extends AbstractDevelopmentService implements RefExportService { + + protected abstract > RefExportOperation createRefExportOperation(); + + @Override + public > RefExportOperation getRefExportOperation() { + return getOrCreate(this::createRefExportOperation, RefExportOperation.class); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefImportService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefImportService.java new file mode 100644 index 000000000..cd5bd1899 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefImportService.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; + + +public abstract class AbstractRefImportService extends AbstractDevelopmentService implements RefImportService { + + protected abstract > RefImportOperation createRefImportOperation(); + + @Override + public > RefImportOperation getRefImportOperation() { + return getOrCreate(this::createRefImportOperation, RefImportOperation.class); + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefQueryService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefQueryService.java new file mode 100644 index 000000000..9430c877d --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/AbstractRefQueryService.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryJumpUrlOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryOperation; + + +public abstract class AbstractRefQueryService extends AbstractDevelopmentService implements RefQueryService { + + protected abstract RefQueryJumpUrlOperation createRefQueryOperation(); + + @Override + public RefQueryJumpUrlOperation getRefQueryOperation() { + return getOrCreate(this::createRefQueryOperation, RefQueryJumpUrlOperation.class); + } + + @Override + public RefQueryOperation getExtraRefQueryOperation(Class clazz) { + return getOrCreate(() -> createExtraRefQueryOperation(clazz), clazz); + } + + protected RefQueryOperation createExtraRefQueryOperation(Class clazz) { + return null; + } +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/DevelopmentService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/DevelopmentService.java new file mode 100644 index 000000000..3b3b9e694 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/DevelopmentService.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppIntegrationService; +import java.util.List; + +public interface DevelopmentService extends AppIntegrationService { + + void setAppStandard(DevelopmentIntegrationStandard appStandard); + + DevelopmentIntegrationStandard getAppStandard(); + + /** + * Labels by default. + * @return All default labels + */ + List getLabels(); +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefCRUDService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefCRUDService.java new file mode 100644 index 000000000..e293ec2cb --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefCRUDService.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCopyOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCreationOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.CopyRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.DSSJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.UpdateRequestRef; + +/** + * Job 管理规范,主要用于管理第三方 AppConn 的 Job(命名为 refJob)。 + *
+ * 建议直接继承 AbstractRefCRUDService + */ +public interface RefCRUDService extends DevelopmentService { + + /** + * 第三方应用工具的 Job(命名为 refJob)的创建操作。 + * @param DSSJobContentRequestRef 的实现类 + * @return RefCreationOperation 的实现类 + */ + > RefCreationOperation getRefCreationOperation(); + + /** + * 第三方应用工具的 Job(命名为 refJob)的复制操作。 + * @param CopyRequestRef 的实现类 + * @return RefCopyOperation 的实现类 + */ + > RefCopyOperation getRefCopyOperation(); + + /** + * 第三方应用工具的 Job(命名为 refJob)的更新操作。 + * @param UpdateRequestRef 的实现类 + * @return RefUpdateOperation 的实现类 + */ + > RefUpdateOperation getRefUpdateOperation(); + + /** + * 第三方应用工具的 Job(命名为 refJob)的删除操作。 + * @param RefJobContentRequestRef 的实现类 + * @return RefDeletionOperation 的实现类 + */ + > RefDeletionOperation getRefDeletionOperation(); + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefExecutionService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefExecutionService.java new file mode 100644 index 000000000..040c0e26c --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefExecutionService.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; + +/** + * Job 执行规范,主要用于执行第三方应用工具的 Job。 + * 建议继承 AbstractRefExecutionService。 + */ +public interface RefExecutionService extends DevelopmentService { + + /** + * 执行第三方 AppConn 的 refJob。 + * @return RefExecutionOperation 实现类 + */ + RefExecutionOperation getRefExecutionOperation(); + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefExportService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefExportService.java new file mode 100644 index 000000000..966acd5e6 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefExportService.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentRequestRef; + +/** + * Job 导出规范,主要用于导出第三方应用工具的 Job。 + * 建议直接继承 AbstractRefExportService。 + */ +public interface RefExportService extends DevelopmentService { + + /** + * 支持将第三方 AppConn 的 Job 导出成 Linkis BML 物料或 InputStream 字节流。 + * @param RefJobContentRequestRef 实现类 + * @return RefExportOperation 实现类 + */ + > RefExportOperation getRefExportOperation(); +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefImportService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefImportService.java new file mode 100644 index 000000000..f8615052e --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefImportService.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; + +/** + * Job 导入规范,主要用于导入第三方应用工具的 Job。 + * 建议直接继承使用 {@code AbstractRefImportService}。 + */ +public interface RefImportService extends DevelopmentService { + + /** + * 通过传入 Linkis BML 物料或 InputStream 字节流,第三方 AppConn 需支持将其转换成一个 refJob。 + * @param ImportRequestRef 实现类 + * @return RefImportOperation 实现类 + */ + > RefImportOperation getRefImportOperation(); + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefQueryService.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefQueryService.java new file mode 100644 index 000000000..8db6f46ab --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/service/RefQueryService.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryJumpUrlOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryOperation; + +/** + * Job 查询规范,主要用于打开第三方应用工具的 Job 的页面。 + */ +public interface RefQueryService extends DevelopmentService { + + /** + * 用于当用户在前端双击该工作流节点时,返回一个可以跳转的 jumpURL + * @return RefQueryJumpUrlOperation 实现类 + */ + RefQueryJumpUrlOperation getRefQueryOperation(); + + /** + * 在 refProject 下获取第三方 AppConn 的 refJob 的一些信息。 + * DSS 框架允许用户实现多个 RefQueryOperation,用于查询 refJob 的个性化信息。 + * @param clazz RefQueryOperation 实现类的 class 对象 + * @return clazz 所代表的实例 + */ + RefQueryOperation getExtraRefQueryOperation(Class clazz); + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/AbstractDevelopmentIntegrationStandard.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/AbstractDevelopmentIntegrationStandard.java new file mode 100644 index 000000000..cc2e94334 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/AbstractDevelopmentIntegrationStandard.java @@ -0,0 +1,80 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.standard; + +import com.webank.wedatasphere.dss.standard.app.development.service.*; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.core.AbstractAppIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; + +import java.io.IOException; + +public abstract class AbstractDevelopmentIntegrationStandard + extends AbstractAppIntegrationStandard implements DevelopmentIntegrationStandard { + + @Override + protected void initService(T service) { + service.setAppStandard(this); + } + + @Override + public RefCRUDService getRefCRUDService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createRefCRUDService, RefCRUDService.class); + } + + protected abstract RefCRUDService createRefCRUDService(); + + @Override + public RefExecutionService getRefExecutionService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createRefExecutionService, RefExecutionService.class); + } + + protected abstract RefExecutionService createRefExecutionService(); + + @Override + public RefExportService getRefExportService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createRefExportService, RefExportService.class); + } + + protected abstract RefExportService createRefExportService(); + + @Override + public RefImportService getRefImportService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createRefImportService, RefImportService.class); + } + + protected abstract RefImportService createRefImportService(); + + @Override + public RefQueryService getRefQueryService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createRefQueryService, RefQueryService.class); + } + + protected abstract RefQueryService createRefQueryService(); + + + @Override + public void init() throws AppStandardErrorException { + + } + + @Override + public void close() throws IOException { + + } +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/DevelopmentIntegrationStandard.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/DevelopmentIntegrationStandard.java new file mode 100644 index 000000000..6ce53c12e --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/DevelopmentIntegrationStandard.java @@ -0,0 +1,87 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.standard; + + +import com.webank.wedatasphere.dss.standard.app.development.service.*; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.core.AppIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +/** + * DSS 开发流程规范。用于打通并关联 DSS 的 Job 与集成的第三方 AppConn 的一个 Job,并在 DSS 的编排器(如:DSS 工作流)中对第三方 + * AppConn 的 Job 进行统一管理,DSS 编排器会提供通用的从需求 -> 设计 -> 开发 -> 调试 -> 导出 -> 导入 -> 发布 的全流程数据应用 + * 开发管理能力。 + *
+ * 因而,该规范主要是为了使集成到 DSS 的第三方 AppConn 具备 DSS 的数据应用全流程开发管理能力,而要求第三方 AppConn 必须实现的各类能力。 + *
+ * 建议直接继承 AbstractDevelopmentIntegrationStandard,包含了五个需要用户实现的 DevelopmentService。 + */ +public interface DevelopmentIntegrationStandard extends AppIntegrationStandard { + + /** + * Job 管理规范,主要用于管理第三方应用工具的 Job(命名为 refJob)。 + * @param appInstance 通过传入 AppInstance 来获取一个 Operation 实例 + * @return RefCRUDService 的实现类 + */ + RefCRUDService getRefCRUDService(AppInstance appInstance); + + /** + * Job 执行规范,主要用于执行第三方应用工具的 Job。 + * @param appInstance 通过传入 AppInstance 来获取一个 Operation 实例 + * @return RefExecutionService 的实现类 + */ + RefExecutionService getRefExecutionService(AppInstance appInstance); + + /** + * Job 导出规范,主要用于导出第三方应用工具的 Job。 + * @param appInstance 通过传入 AppInstance 来获取一个 Operation 实例 + * @return RefExportService 的实现类 + */ + RefExportService getRefExportService(AppInstance appInstance); + + /** + * Job 导入规范,主要用于导入第三方应用工具的 Job。 + * @param appInstance 通过传入 AppInstance 来获取一个 Operation 实例 + * @return RefImportService 的实现类 + */ + RefImportService getRefImportService(AppInstance appInstance); + + /** + * Job 查询规范,主要用于打开第三方应用工具的 Job 的页面。 + * @param appInstance 通过传入 AppInstance 来获取一个 Operation 实例 + * @return RefQueryService 的实现类 + */ + RefQueryService getRefQueryService(AppInstance appInstance); + + + @Override + default String getStandardName() { + return "developmentIntegrationStandard"; + } + + @Override + default int getGrade() { + return 3; + } + + @Override + default boolean isNecessary() { + return false; + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/OnlyExecutionDevelopmentStandard.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/OnlyExecutionDevelopmentStandard.java new file mode 100644 index 000000000..4bb79270e --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/standard/OnlyExecutionDevelopmentStandard.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.development.standard; + +import com.webank.wedatasphere.dss.standard.app.development.service.*; + + +public abstract class OnlyExecutionDevelopmentStandard extends AbstractDevelopmentIntegrationStandard { + + @Override + protected RefCRUDService createRefCRUDService() { + return null; +} + + @Override + protected RefExportService createRefExportService() { + return null; + } + + @Override + protected RefImportService createRefImportService() { + return null; + } + + @Override + protected RefQueryService createRefQueryService() { + return null; + } + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/DSSJobContentConstant.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/DSSJobContentConstant.java new file mode 100644 index 000000000..99c581732 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/DSSJobContentConstant.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.standard.app.development.utils; + +/** + * @author enjoyyin + * @date 2022-03-15 + * @since 0.5.0 + */ +public class DSSJobContentConstant { + + /** + * DSSJob 所归属的 DSS编排(如:DSS工作流)的编排名,其对应的 value 值为 {@code String}。 + */ + public static final String ORCHESTRATION_NAME = "orchestrationName"; + + /** + * DSSJob 所归属的 DSS编排(如:DSS工作流)的编排名,其对应的 value 值为 {@code Long}。 + */ + public static final String ORCHESTRATION_ID = "orchestrationId"; + + /** + * DSS编排的版本号,例如: DSS 工作流所在的版本。 + * 请注意:该 key 只有在创建 refJob 时才存在,更新等操作不存在该 key。 + * 强烈建议根据 orcVersion 提供的 value 值,为 refJob 命名。 + * 例如:
+ * DSSJobContentRequestRef 的 name 为 test,orcVersion 为 v2,建议第三方 AppConn 在创建一个 refJob 时, + * 新的 refJob 命名为 test_v2。 + *
+ * 其对应的 value 值为 {@code String},如:v1、v2、v3。 + */ + public static final String ORC_VERSION_KEY = "orcVersion"; + + /** + * DSSJob 的描述,其对应的 value 值为 {@code String}。 + */ + public static final String NODE_DESC_KEY = "desc"; + + /** + * DSSJob 所依赖的上游工作流节点,其对应的 value 值为 {@code List}。 + */ + public static final String UP_STREAM_KEY = "upStreams"; + + public static final String UP_STREAM_EMPTY_VALUE = "empty"; + +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/DevelopmentOperationUtils.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/DevelopmentOperationUtils.java new file mode 100644 index 000000000..4537337bf --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/DevelopmentOperationUtils.java @@ -0,0 +1,112 @@ +package com.webank.wedatasphere.dss.standard.app.development.utils; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.standard.app.development.operation.DevelopmentOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.*; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import com.webank.wedatasphere.dss.standard.common.utils.RequestRefUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.*; + +/** + * @author enjoyyin + * @date 2022-03-12 + * @since 0.5.0 + */ +public class DevelopmentOperationUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(DevelopmentOperationUtils.class); + + public static V tryDevelopmentOperation(Supplier getDevelopmentService, + Function getDevelopmentOperation, + Consumer jobContentRequestRefConsumer, + Consumer refJobContentRequestRefConsumer, + Consumer contextRequestRefConsumer, + Consumer projectRefRequestRefConsumer, + BiFunction requestRefOperationFunction, + BiConsumer responseRefConsumer, + String errorMsg) { + DevelopmentService developmentService = getDevelopmentService.get(); + if(developmentService == null) { + LOGGER.error("{} failed. Caused by: The AppConn has no DevelopmentService.", errorMsg); + throw new ExternalOperationWarnException(60092, errorMsg + " failed. Caused by: The AppConn has no DevelopmentService."); + } + DevelopmentOperation operation = getDevelopmentOperation.apply(developmentService); + if(operation == null) { + LOGGER.error("{} failed. Caused by: The corresponding development operation is not exists.", errorMsg); + throw new ExternalOperationWarnException(60092, errorMsg + " failed. Caused by: The AppConn has no corresponding development operation."); + } + K requestRef = RequestRefUtils.getRequestRef(operation); + if(jobContentRequestRefConsumer != null && requestRef instanceof DSSJobContentRequestRef) { + jobContentRequestRefConsumer.accept((DSSJobContentRequestRef) requestRef); + } + if(refJobContentRequestRefConsumer != null && requestRef instanceof RefJobContentRequestRef) { + refJobContentRequestRefConsumer.accept((RefJobContentRequestRef) requestRef); + } + if(contextRequestRefConsumer != null && requestRef instanceof DSSContextRequestRef) { + contextRequestRefConsumer.accept((DSSContextRequestRef) requestRef); + } + if(projectRefRequestRefConsumer != null && requestRef instanceof ProjectRefRequestRef) { + projectRefRequestRefConsumer.accept((ProjectRefRequestRef) requestRef); + } + V responseRef = requestRefOperationFunction.apply(operation, requestRef); + if(responseRef.isFailed()) { + LOGGER.error("{} failed. Caused by: {}.", errorMsg, responseRef.getErrorMsg()); + DSSExceptionUtils.dealWarnException(61123, + String.format("%s failed. Caused by: %s.", errorMsg, responseRef.getErrorMsg()), + ExternalOperationWarnException.class); + } + if(responseRefConsumer != null) { + responseRefConsumer.accept(requestRef, responseRef); + } + return responseRef; + } + + public static V tryDevelopmentOperation(Supplier getDevelopmentService, + Function getDevelopmentOperation, + Consumer jobContentRequestRefConsumer, + Consumer refJobContentRequestRefConsumer, + Consumer contextRequestRefConsumer, + Consumer projectRefRequestRefConsumer, + BiFunction requestRefOperationFunction, + String errorMsg) { + return tryDevelopmentOperation(getDevelopmentService, getDevelopmentOperation, jobContentRequestRefConsumer, refJobContentRequestRefConsumer, + contextRequestRefConsumer, projectRefRequestRefConsumer, requestRefOperationFunction, null, errorMsg); + } + + public static V tryRefJobContentRequestRefOperation(Supplier getDevelopmentService, + Function getDevelopmentOperation, + Consumer refJobContentRequestRefConsumer, + Consumer contextRequestRefConsumer, + Consumer projectRefRequestRefConsumer, + BiFunction responseRefConsumer, + String errorMsg) { + return tryDevelopmentOperation(getDevelopmentService, getDevelopmentOperation, null, + refJobContentRequestRefConsumer, contextRequestRefConsumer, projectRefRequestRefConsumer, responseRefConsumer, errorMsg); + } + + public static V tryDSSJobContentRequestRefOperation(Supplier getDevelopmentService, + Function getDevelopmentOperation, + Consumer jobContentRequestRefConsumer, + Consumer contextRequestRefConsumer, + Consumer projectRefRequestRefConsumer, + BiFunction responseRefConsumer, + String errorMsg) { + return tryDevelopmentOperation(getDevelopmentService, getDevelopmentOperation, jobContentRequestRefConsumer, + null, contextRequestRefConsumer, projectRefRequestRefConsumer, responseRefConsumer, errorMsg); + } + + public static V tryDevelopmentRequestRefOperation(Supplier getDevelopmentService, + Function getDevelopmentOperation, + Consumer contextRequestRefConsumer, + Consumer projectRefRequestRefConsumer, + BiFunction responseRefConsumer, + String errorMsg) { + return tryDevelopmentOperation(getDevelopmentService, getDevelopmentOperation, null, + null, contextRequestRefConsumer, projectRefRequestRefConsumer, responseRefConsumer, errorMsg); + } +} diff --git a/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/QueryJumpUrlConstant.java b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/QueryJumpUrlConstant.java new file mode 100644 index 000000000..f320b4bb8 --- /dev/null +++ b/dss-standard/development-standard/development-process-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/development/utils/QueryJumpUrlConstant.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.standard.app.development.utils; + +import org.apache.linkis.protocol.util.ImmutablePair; + +/** + * @author enjoyyin + * @date 2022-04-14 + * @since 1.1.0 + */ +public class QueryJumpUrlConstant { + + /** + * 工作流节点 ID,对应到 DSS 工作流的 nodes 里面对应 node 的 key。 + */ + public static final ImmutablePair NODE_ID = new ImmutablePair<>("nodeId", "${nodeId}"); + + /** + * 工作流节点 ID,对应到 DSS 工作流的 nodes 里面对应 node 的 name。 + */ + public static final ImmutablePair NODE_NAME = new ImmutablePair<>("nodeName", "${nodeName}"); + + /** + * DSS 工程名。 + */ + public static final ImmutablePair PROJECT_NAME = new ImmutablePair<>("projectName", "${projectName}"); + +} diff --git a/dss-standard/dss-standard-common/pom.xml b/dss-standard/dss-standard-common/pom.xml new file mode 100644 index 000000000..f82d4f624 --- /dev/null +++ b/dss-standard/dss-standard-common/pom.xml @@ -0,0 +1,98 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../pom.xml + + 4.0.0 + + dss-standard-common + + + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + **/*.yml + **/*.properties + **/*.sh + **/log4j2.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/resources + + + ${project.artifactId}-${project.version} + + + + + product + + 1.0.0-SNAPSHOT + + + + + \ No newline at end of file diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppIntegrationService.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppIntegrationService.java new file mode 100644 index 000000000..268a5b237 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppIntegrationService.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.app; + +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.service.AppService; + + +public interface AppIntegrationService extends AppService { + + void setSSORequestService(T ssoRequestService); + + T getSSORequestService(); + + AppInstance getAppInstance(); + + void setAppInstance(AppInstance appInstance); + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppIntegrationServiceImpl.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppIntegrationServiceImpl.java new file mode 100644 index 000000000..04c1dc798 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppIntegrationServiceImpl.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.app; + + +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.service.AppService; +import com.webank.wedatasphere.dss.standard.common.service.AppServiceImpl; + + +public class AppIntegrationServiceImpl extends AppServiceImpl implements AppIntegrationService { + + private T ssoRequestService; + private AppInstance appInstance; + + @Override + public void setSSORequestService(T ssoRequestService) { + this.ssoRequestService = ssoRequestService; + } + + @Override + public T getSSORequestService() { + return ssoRequestService; + } + + @Override + public AppInstance getAppInstance() { + return appInstance; + } + + @Override + public void setAppInstance(AppInstance appInstance) { + this.appInstance = appInstance; + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppSingletonIntegrationServiceImpl.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppSingletonIntegrationServiceImpl.java new file mode 100644 index 000000000..b2c13f4df --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/app/AppSingletonIntegrationServiceImpl.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.app; + +import com.webank.wedatasphere.dss.standard.common.service.AppService; +import com.webank.wedatasphere.dss.standard.common.service.Operation; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; + + +public class AppSingletonIntegrationServiceImpl extends AppIntegrationServiceImpl { + + private final List appOperations = new ArrayList<>(); + + protected TO getOrCreate(Supplier create, Class clazz) { + Supplier createAndPut = () -> { + TO t = create.get(); + if(t == null) { + return null; + } + initOperation(t); + t.init(); + appOperations.add(t); + return t; + }; + Supplier> filterOperation = () -> appOperations.stream().filter(clazz::isInstance).findFirst().map(operation -> (TO) operation); + return filterOperation.get().orElseGet(() -> { + synchronized (appOperations) { + return filterOperation.get().orElseGet(createAndPut); + } + }); + } + + protected void initOperation(O operation) {} + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AbstractAppIntegrationStandard.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AbstractAppIntegrationStandard.java new file mode 100644 index 000000000..2cc8b7648 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AbstractAppIntegrationStandard.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.core; + +import com.webank.wedatasphere.dss.standard.common.app.AppIntegrationService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.service.AppService; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + + +public abstract class AbstractAppIntegrationStandard + implements AppIntegrationStandard { + + protected final Map> appServices = new HashMap<>(); + protected SSORequestService ssoRequestService; + + protected T getOrCreate(AppInstance appInstance, Supplier create, Class clazz) { + Supplier createAndPut = () -> { + T t = create.get(); + if(t == null) { + return null; + } + t.setSSORequestService(ssoRequestService); + t.setAppInstance(appInstance); + initService(t); + appServices.get(appInstance).add(t); + return t; + }; + if(!appServices.containsKey(appInstance)) { + synchronized (appServices) { + if(!appServices.containsKey(appInstance)) { + appServices.put(appInstance, new ArrayList<>()); + return createAndPut.get(); + } + } + } + final List services = appServices.get(appInstance); + Supplier> filterService = () -> services.stream().filter(clazz::isInstance).findFirst().map(service -> (T) service); + return filterService.get().orElseGet(() -> { + synchronized (services) { + return filterService.get().orElseGet(createAndPut); + } + }); + } + + protected void initService(T service) {} + + @Override + public void setSSORequestService(SSORequestService ssoRequestService) { + this.ssoRequestService = ssoRequestService; + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AppIntegrationStandard.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AppIntegrationStandard.java new file mode 100644 index 000000000..8fbe27bae --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AppIntegrationStandard.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.core; + + +import com.webank.wedatasphere.dss.standard.common.service.AppService; + + +public interface AppIntegrationStandard extends AppStandard { + + void setSSORequestService(SSORequestService ssoService); + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AppStandard.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AppStandard.java new file mode 100644 index 000000000..c19af5297 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/core/AppStandard.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.core; + + +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; + +import java.io.Closeable; + + +public interface AppStandard extends Closeable { + + String getStandardName(); + + int getGrade(); + + void init() throws AppStandardErrorException; + + /** + * 是否必须 + * @return true 表示必须,false表示非必须标准 + */ + default boolean isNecessary() { + return false; + } + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppDesc.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppDesc.java new file mode 100644 index 000000000..c507e3f83 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppDesc.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.desc; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.common.exception.NoSuchAppInstanceException; + +import java.util.List; + + +public interface AppDesc { + + String getAppName(); + + List getAppInstances(); + + /** + * 对于每个 AppConn 而言,允许拥有多个 AppInstance,如需要获取相应的 AppInstance, + * 通过传入的 labels 来与 AppInstance 的 getLabels() 所对应的 labels 来进行匹配。
+ * 返回的 List 实例,其中第0个是匹配程度最大的实例。 + * */ + List getAppInstancesByLabels(List labels) throws NoSuchAppInstanceException; + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppDescImpl.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppDescImpl.java new file mode 100644 index 000000000..5bbb43786 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppDescImpl.java @@ -0,0 +1,100 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.desc; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.common.exception.NoSuchAppInstanceException; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + + +public class AppDescImpl implements AppDesc { + + private static final Logger LOG = LoggerFactory.getLogger(AppDescImpl.class); + + private String appName; + private List appInstances = new ArrayList<>(); + + @Override + public String getAppName() { + return appName; + } + + public void setAppName(String appName) { + this.appName = appName; + } + + @Override + public List getAppInstances() { + return appInstances; + } + + @Override + public List getAppInstancesByLabels(List labels) throws NoSuchAppInstanceException{ + // 1. 通过用户自定义的比较器完成的排序,返回最前一个appInstance + // 2. 返回与labels完全匹配的appInstance + AppInstance targetAppInstance = null; + int similarity = 0; + int maxSimilarity = 0; + for(AppInstance appInstance: appInstances) { + similarity = 0; + List targetLabels = appInstance.getLabels(); + for(DSSLabel label: targetLabels) { + for(DSSLabel userLabel: labels) { + // if user's label equal the target label, then the similarity increase. + if (isEqualLabel(userLabel,label)) { + similarity++; + } + // if current similarity bigger than the maxSimilarity. + if (similarity > maxSimilarity) { + maxSimilarity = similarity; + targetAppInstance = appInstance; + } + } + } + } + // if all the labels is different form the target, then return null. + if(maxSimilarity <= 0) { + LOG.error("{} has no such AppInstance machs the labels: {}.", appName, labels); + throw new NoSuchAppInstanceException(60002, "No such AppInstance machs the labels."); + } + return Collections.singletonList(targetAppInstance); + } + + /** + * 判断两个label是否相等 + */ + public boolean isEqualLabel(DSSLabel userLabel, DSSLabel label){ + String userEnv = userLabel.getValue().get(userLabel.getLabelKey()); + String instanceEnv = label.getValue().get(label.getLabelKey()); + return StringUtils.isNotBlank(userEnv) && userEnv.equalsIgnoreCase(instanceEnv); + } + + public void addAppInstance(AppInstance appInstance) { + appInstances.add(appInstance); + } + + public void removeAppInstance(AppInstance appInstance) { + appInstances.remove(appInstance); + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppInstance.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppInstance.java new file mode 100644 index 000000000..1bfa01405 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppInstance.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.desc; +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; +import java.util.Map; + + +public interface AppInstance { + + Long getId(); + + /** + * 返回第三方 AppConn 的基础URL,例如:http://ip:port/ 。是 DSS 接下来访问这个第三方 AppConn 的基础 URL, + * AppConn 的 Operation 都将通过这个 baseURL 去拼接实际请求的具体 URL。 + * @return 返回第三方 AppConn 的基础URL + */ + String getBaseUrl(); + + /** + * 这里返回的是第三方 AppConn 主页的 URI,即不包含 IP 和 Port 部分的 URI 内容,如想获取完整的 + * homepage URL,请通过 getBaseUrl() + getHomepageUri()获取。 + * 请注意:DSS 允许用户在 getHomepageUri() 中携带 workspaceId 和 workspaceName 变量,DSS会在使用时自动替换, + * 参考的格式如下:
+ * #/home?workspaceId=${workspaceId}&workspaceName=${workspaceName} + * @return 返回第三方 AppConn 主页的 URI,例如:#/home + */ + String getHomepageUri(); + + /** + * 与 dss_appinstance 数据库表的 enhance_json 字段对应,允许用户在数据库表中传入一个 json 格式的参数列表, + * 第三方应用 AppConn 的 Operation 可以直接调用该方法获取 enhance_json 里面的参数列表。
+ * 当然,您也可以通过该 AppConn 的 appconn.properties 设置一些所需的参数。appconn.properties 文件存放在 + * ${APPCONN_HOME}/${APPCONN_NAME}/conf 目录下。
+ * 目前 DSS 本身需要的参数如下: + * 1. reqUri:(非必须) 如果第三方应用集成了 DSS 一级规范,当在 DSS 的顶部菜单栏点击跳转到第三方应用的首页时,需要用到 + * 该属性。 reqUri 用于指定一个 RESTFul 请求的 URI,该 URI 可访问第三方系统后台的某个 RESTFUL 接口(随意的 Restful 接口), + * DSS 放置在第三方系统的一级规范 Jar 包会自动拦截该请求,加上用户态后自动重定向给实际的前端首页。更多请参考 DSS 一级规范。 + * @return 第三方 AppConn 参数列表 + */ + Map getConfig(); + + List getLabels(); + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppInstanceImpl.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppInstanceImpl.java new file mode 100644 index 000000000..dfcbe422f --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/desc/AppInstanceImpl.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.desc; + +import com.google.common.base.Objects; +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; +import java.util.Map; + + +public class AppInstanceImpl implements AppInstance { + + private Long id; + private String baseUrl; + private String homepageUri; + private Map config; + private List labels; + + @Override + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Override + public String getBaseUrl() { + return baseUrl; + } + + @Override + public String getHomepageUri() { + return homepageUri; + } + + public void setHomepageUri(String homepageUri) { + this.homepageUri = homepageUri; + } + + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + @Override + public Map getConfig() { + return config; + } + + public void setConfig(Map config) { + this.config = config; + } + + @Override + public List getLabels() { + return labels; + } + + public void setLabels(List labels) { + this.labels = labels; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AppInstanceImpl that = (AppInstanceImpl) o; + return Objects.equal(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hashCode(id); + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/AsyncResponseRef.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/AsyncResponseRef.java new file mode 100644 index 000000000..40613bd0e --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/AsyncResponseRef.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.entity.ref; + +import java.util.function.Consumer; + +/** + * This is the topper interface of async responseRef, if one operation want to provide async capability, + * please provides a child interface which extends this super interface to use. + * Notice: all detail classes should never directly implement this topper interface, since every operation will + * provide a child interface for async operation. + */ +public interface AsyncResponseRef extends ResponseRef { + + long getStartTime(); + + default boolean isCompleted() { + return isFailed() || isSucceed(); + } + + ResponseRef getResponse(); + + void waitForCompleted() throws InterruptedException; + + void notifyMe(Consumer notifyListener); + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/InternalResponseRef.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/InternalResponseRef.java new file mode 100644 index 000000000..641badfa9 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/InternalResponseRef.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.standard.common.entity.ref; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-06 + * @since 0.5.0 + */ +public class InternalResponseRef extends ResponseRefImpl { + + private Map data; + private String message; + + public InternalResponseRef(String responseBody, int status, String errorMsg, + Map responseMap) { + super(responseBody, status, errorMsg, responseMap); + this.data = (Map) responseMap.get("data"); + this.message = (String) responseMap.get("message"); + } + + public Map getData() { + return data; + } + + public String getMessage() { + return message; + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/Ref.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/Ref.java new file mode 100644 index 000000000..6be5749e9 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/Ref.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.entity.ref; + +public interface Ref { + + @Override + boolean equals(Object ref); + + @Override + String toString(); + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/RequestRef.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/RequestRef.java new file mode 100644 index 000000000..c9ba5c8bf --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/RequestRef.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.entity.ref; +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + +public interface RequestRef extends Ref { + + Object getParameter(String key); + + void setParameter(String key, Object value); + + Map getParameters(); + + String getName(); + + String getType(); + + default List getDSSLabels(){ + return new ArrayList<>(); + } + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/RequestRefImpl.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/RequestRefImpl.java new file mode 100644 index 000000000..1d7d4d91b --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/RequestRefImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.entity.ref; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class RequestRefImpl implements RequestRef { + + private Map parameters = new HashMap<>(); + protected String name; + protected String type; + protected List dssLabels; + + @Override + public String getName() { + return name; + } + + @Override + public String getType() { + return type; + } + + @Override + public Object getParameter(String key) { + return parameters.get(key); + } + + @Override + public void setParameter(String key, Object value) { + parameters.put(key, value); + } + + @Override + public Map getParameters() { + return parameters; + } + + @Override + public List getDSSLabels() { + return dssLabels; + } + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRef.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRef.java new file mode 100644 index 000000000..81004891d --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRef.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.entity.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder.*; + +import java.util.Map; + + +public interface ResponseRef extends Ref { + + Object getValue(String key); + + Map toMap(); + + String getResponseBody(); + + int getStatus(); + + String getErrorMsg(); + + default boolean isSucceed() { + return getStatus() == 0 || getStatus() == 200; + } + + default boolean isFailed() { + return !isSucceed(); + } + + static InternalResponseRefBuilder newInternalBuilder() { + return new InternalResponseRefBuilder(); + } + + static ExternalResponseRefBuilder newExternalBuilder() { + return new ExternalResponseRefBuilder(); + } + + static MutableExternalResponseRefBuilder newMutableExternalBuilder() { + return new MutableExternalResponseRefBuilder(); + } + +} \ No newline at end of file diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRefBuilder.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRefBuilder.java new file mode 100644 index 000000000..5f3993b2f --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRefBuilder.java @@ -0,0 +1,149 @@ +package com.webank.wedatasphere.dss.standard.common.entity.ref; + +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; + +import java.util.HashMap; +import java.util.Map; + + +public interface ResponseRefBuilder, T extends ResponseRef> { + + T build(); + + T success(); + + T error(String errorMsg); + + T error(Throwable cause); + + class InternalResponseRefBuilder, T extends InternalResponseRef> + implements ResponseRefBuilder { + + protected String responseBody; + + public R setResponseBody(String responseBody) { + this.responseBody = responseBody; + return (R) this; + } + + protected T createResponseRef(String responseBody, int status, + String errorMsg, Map responseMap) { + return (T) new InternalResponseRef(responseBody, status, errorMsg, responseMap); + } + + @Override + public final T build() { + Map responseMap = DSSCommonUtils.COMMON_GSON.fromJson(responseBody, Map.class); + int status = (int) DSSCommonUtils.parseToLong(responseMap.get("status")); + String errorMsg = (String) responseMap.get("message"); + return createResponseRef(responseBody, status, errorMsg, responseMap); + } + + @Override + public T success() { + return createResponseRef("", 200, null, new HashMap<>(0)); + } + + @Override + public T error(String errorMsg) { + return createResponseRef("", 400, errorMsg, new HashMap<>(0)); + } + + @Override + public T error(Throwable cause) { + return error(ExceptionUtils.getRootCauseMessage(cause)); + } + } + + class ExternalResponseRefBuilder, T extends ResponseRef> + implements ResponseRefBuilder { + protected String responseBody; + protected int status; + protected String errorMsg; + protected Map responseMap; + + public R setResponseBody(String responseBody) { + this.responseBody = responseBody; + return (R) this; + } + + public R setStatus(int status) { + this.status = status; + return (R) this; + } + + public R setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + return (R) this; + } + + public R setResponseMap(Map responseMap) { + this.responseMap = responseMap; + return (R) this; + } + + @Override + public T success() { + this.status = 200; + if(responseBody == null) { + this.responseBody = ""; + } + if(responseMap == null) { + this.responseMap = new HashMap<>(0); + } + return build(); + } + + @Override + public T error(String errorMsg) { + this.status = 400; + if(responseBody == null) { + this.responseBody = ""; + } + if(responseMap == null) { + this.responseMap = new HashMap<>(0); + } + this.errorMsg = errorMsg; + return build(); + } + + @Override + public T error(Throwable cause) { + return error(ExceptionUtils.getRootCauseMessage(cause)); + } + + protected T createResponseRef() { + return (T) new ResponseRefImpl(responseBody, status, errorMsg, responseMap); + } + + @Override + public T build() { + if(responseMap == null && StringUtils.isNotBlank(responseBody)) { + responseMap = DSSCommonUtils.COMMON_GSON.fromJson(responseBody, Map.class); + } else if(responseMap == null) { + responseMap = new HashMap<>(); + } + return createResponseRef(); + } + } + + class MutableExternalResponseRefBuilder, T extends ResponseRef> + extends ExternalResponseRefBuilder { + + public R extendsResponseBody() { + responseMap = DSSCommonUtils.COMMON_GSON.fromJson(responseBody, Map.class); + return (R) this; + } + + public R addResponse(String key, Object value){ + if(responseMap == null) { + responseMap = new HashMap<>(); + } + responseMap.put(key, value); + return (R) this; + } + + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRefImpl.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRefImpl.java new file mode 100644 index 000000000..b4786efd6 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/entity/ref/ResponseRefImpl.java @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.dss.standard.common.entity.ref; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-03 + * @since 0.5.0 + */ +public class ResponseRefImpl implements ResponseRef { + + protected String responseBody; + protected int status = -1; + protected String errorMsg; + protected Map responseMap; + + public ResponseRefImpl(String responseBody, int status, String errorMsg, Map responseMap) { + this.responseBody = responseBody; + this.status = status; + this.errorMsg = errorMsg; + this.responseMap = responseMap; + } + + @Override + public Map toMap() { + return responseMap; + } + + @Override + public Object getValue(String key) { + if(responseMap == null) { + return null; + } else { + return responseMap.get(key); + } + } + + @Override + public String getResponseBody() { + return responseBody; + } + + @Override + public int getStatus() { + return status; + } + + @Override + public String getErrorMsg() { + return errorMsg; + } + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/AppStandardErrorException.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/AppStandardErrorException.java new file mode 100644 index 000000000..5e469e2d7 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/AppStandardErrorException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; + + +public class AppStandardErrorException extends DSSRuntimeException { + + public AppStandardErrorException(int errorCode, String errorMessage){ + super(errorCode, errorMessage); + } + + public AppStandardErrorException(int errorCode, String message, Throwable cause) { + super(errorCode, message, cause); + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/AppStandardWarnException.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/AppStandardWarnException.java new file mode 100644 index 000000000..4e64bff5b --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/AppStandardWarnException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.exception; + + +import org.apache.linkis.common.exception.WarnException; + +public class AppStandardWarnException extends WarnException { + + public AppStandardWarnException(int errorCode, String message) { + super(errorCode, message); + } + + public AppStandardWarnException(int errorCode, String message, Throwable cause) { + super(errorCode, message); + initCause(cause); + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/NoSuchAppInstanceException.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/NoSuchAppInstanceException.java new file mode 100644 index 000000000..5e1668dbf --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/NoSuchAppInstanceException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + + +public class NoSuchAppInstanceException extends DSSErrorException { + + public NoSuchAppInstanceException(int errCode, String desc) { + super(errCode, desc); + } + + public NoSuchAppInstanceException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/operation/ExternalOperationFailedException.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/operation/ExternalOperationFailedException.java new file mode 100644 index 000000000..940d92ce7 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/operation/ExternalOperationFailedException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.exception.operation; + +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; + +public class ExternalOperationFailedException extends AppStandardErrorException { + + public ExternalOperationFailedException(int errorCode, String message){ + super(errorCode, message); + } + + public ExternalOperationFailedException(int errorCode, String message, Throwable cause) { + super(errorCode, message, cause); + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/operation/ExternalOperationWarnException.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/operation/ExternalOperationWarnException.java new file mode 100644 index 000000000..10d8452af --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/exception/operation/ExternalOperationWarnException.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.standard.common.exception.operation; + +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardWarnException; + +/** + * @author enjoyyin + * @date 2022-03-11 + * @since 0.5.0 + */ +public class ExternalOperationWarnException extends AppStandardWarnException { + + public ExternalOperationWarnException(int errorCode, String message){ + super(errorCode, message); + } + + public ExternalOperationWarnException(int errorCode, String message, Throwable cause) { + super(errorCode, message, cause); + } + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/AppService.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/AppService.java new file mode 100644 index 000000000..330fc01ac --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/AppService.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.service; + + +public interface AppService { + + Operation createOperation(Class clazz); + + boolean isOperationExists(Class clazz); + + boolean isOperationNecessary(Class clazz); + +} \ No newline at end of file diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/AppServiceImpl.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/AppServiceImpl.java new file mode 100644 index 000000000..9495a3760 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/AppServiceImpl.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.service; + +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + + +public class AppServiceImpl implements AppService { + + private Set> necessaryOperations = new HashSet<>(); + + protected void registerNecessaryOperation(Class clazz) { + necessaryOperations.add(clazz); + } + + @Override + public Operation createOperation(Class clazz) { + String clazzSimpleName = clazz.getSimpleName(); + List methodList = Arrays.stream(this.getClass().getDeclaredMethods()) + .filter(method -> method.getReturnType() == clazz + && method.getParameterCount() == 0).collect(Collectors.toList()); + if(methodList.size() == 1) { + try { + return (Operation) methodList.get(0).invoke(this); + } catch (ReflectiveOperationException e) { + throw new ExternalOperationWarnException(80020, "Not exists operation: " + clazzSimpleName, e); + } + } else if(methodList.isEmpty()) { + return notFoundOperation(clazz); + } else { + return multiFoundOperation(clazz); + } + } + + protected Operation notFoundOperation(Class clazz) { + throw new ExternalOperationWarnException(80020, "Not exists operation: " + clazz.getSimpleName()); + } + + protected Operation multiFoundOperation(Class clazz) { + throw new ExternalOperationWarnException(80020, "Multi exists operations: " + clazz.getSimpleName()); + } + + @Override + public boolean isOperationExists(Class clazz) { + try{ + return createOperation(clazz) != null; + } catch (ExternalOperationWarnException e) { + return false; + } + } + + @Override + public boolean isOperationNecessary(Class clazz) { + boolean isNecessary = necessaryOperations.contains(clazz); + if(isNecessary) { + return true; + } else { + return isOperationExists(clazz); + } + } +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/Operation.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/Operation.java new file mode 100644 index 000000000..48979968d --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/service/Operation.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.service; + + +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; + +/** + * Operation是顶层的操作类,对于操作来说,传入一个RequestRef,返回一个ResponseRef + */ +public interface Operation { + + default void init() {} + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/utils/AppStandardClassUtils.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/utils/AppStandardClassUtils.java new file mode 100644 index 000000000..5778e5b28 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/utils/AppStandardClassUtils.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.common.utils; + +import com.webank.wedatasphere.dss.common.utils.ClassUtils.ClassHelper; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; +import org.reflections.Reflections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is defined for AppConn jar, if some classes in AppConn want to load a class in AppConn jar. + */ +public class AppStandardClassUtils extends ClassHelper { + + private static final Map INSTANCES = new HashMap<>(); + private static final Map CLASS_LOADER_MAP = new HashMap<>(); + + private static final Logger LOGGER = LoggerFactory.getLogger(AppStandardClassUtils.class); + + public static ClassLoader getClassLoader(String appConnName, Supplier createClassLoader) { + if(!CLASS_LOADER_MAP.containsKey(appConnName)) { + synchronized (AppStandardClassUtils.class) { + if(!CLASS_LOADER_MAP.containsKey(appConnName)) { + CLASS_LOADER_MAP.put(appConnName, createClassLoader.get()); + LOGGER.info("Has stored {} ClassLoader.", CLASS_LOADER_MAP.size()); + } + } + } + return CLASS_LOADER_MAP.get(appConnName); + } + + public static AppStandardClassUtils getInstance(String appConnName) { + if(!INSTANCES.containsKey(appConnName)) { + synchronized (AppStandardClassUtils.class) { + if(!INSTANCES.containsKey(appConnName)) { + INSTANCES.put(appConnName, new AppStandardClassUtils(appConnName)); + LOGGER.info("Has stored {} AppStandardClassUtils.", INSTANCES.size()); + } + } + } + return INSTANCES.get(appConnName); + } + + private Reflections reflection; + private String appConnName; + + private AppStandardClassUtils(String appConnName) { + this.appConnName = appConnName; + } + + @Override + protected Reflections getReflections(Class clazz) { + if(reflection == null) { + synchronized (this) { + if(reflection == null) { + ClassLoader classLoader = clazz.getClassLoader(); + reflection = new Reflections("com.webank.wedatasphere.dss", CLASS_LOADER_MAP.get(appConnName), classLoader); + } + } + } + return reflection; + } + +} diff --git a/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/utils/RequestRefUtils.java b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/utils/RequestRefUtils.java new file mode 100644 index 000000000..d14cb9be0 --- /dev/null +++ b/dss-standard/dss-standard-common/src/main/java/com/webank/wedatasphere/dss/standard/common/utils/RequestRefUtils.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.dss.standard.common.utils; + +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.stream.Stream; + +/** + * @author enjoyyin + * @date 2022-03-11 + * @since 0.5.0 + */ +public class RequestRefUtils { + + public static T getRequestRef(Object operation) { + ParameterizedType parameterizedType = (ParameterizedType) operation.getClass().getGenericSuperclass(); + return newInstance(operation, parameterizedType); + } + + @Deprecated + public static T getRequestRef(Object operation, Class clazz) { + return Stream.of(operation.getClass().getGenericInterfaces()) + .filter(c -> clazz.isAssignableFrom((Class) c)).map(c -> { + ParameterizedType parameterizedType = (ParameterizedType) c; + return (T) newInstance(operation, parameterizedType); + }).findAny().orElseThrow(() -> new ExternalOperationFailedException(50063, "Cannot find the real requestRef of " + operation.getClass() + ".")); + } + + private static T newInstance(Object operation, ParameterizedType parameterizedType) { + Type[] types = parameterizedType.getActualTypeArguments(); + if(types.length > 0 && types.length <= 2) { + Class t = (Class) types[0]; + try { + // please notice, don't try to use ClassUtils.getInstance() to create it. + return t.newInstance(); + } catch (Exception e) { + throw new ExternalOperationFailedException(50063, "create the instance of " + types[0] + " failed.", e); + } + } else { + throw new ExternalOperationFailedException(50063, "Cannot find the real requestRef of " + operation.getClass().getSimpleName() + "."); + } + } + +} diff --git a/dss-standard/pom.xml b/dss-standard/pom.xml new file mode 100644 index 000000000..6131545d2 --- /dev/null +++ b/dss-standard/pom.xml @@ -0,0 +1,45 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../pom.xml + + 4.0.0 + + dss-standard + pom + + + dss-standard-common + sso-standard/sso-integration-standard + sso-standard/origin-sso-integration-standard + sso-standard/spring-origin-sso-integration-plugin + structure-standard/dss-structure-integration-standard + structure-standard/dss-project-plugin + structure-standard/spring-origin-dss-project-plugin + structure-standard/dss-role-plugin + development-standard/development-process-standard + development-standard/development-process-standard-execution + + + \ No newline at end of file diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml b/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml new file mode 100644 index 000000000..b1a8ffdb6 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml @@ -0,0 +1,76 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-origin-sso-integration-standard + + + + org.apache.linkis + linkis-gateway-httpclient-support + ${linkis.version} + + + javax.servlet + javax.servlet-api + 3.1.0 + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + + + \ No newline at end of file diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/HttpSSOIntegrationStandard.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/HttpSSOIntegrationStandard.scala new file mode 100644 index 000000000..d87eb415f --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/HttpSSOIntegrationStandard.scala @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.dss.standard.app.sso.origin + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient +import com.webank.wedatasphere.dss.standard.app.sso.origin.plugin.OriginSSOPluginServiceImpl +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.HttpSSORequestServiceImpl +import com.webank.wedatasphere.dss.standard.app.sso.plugin.SSOPluginService +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService +import com.webank.wedatasphere.dss.standard.app.sso.user.SSOUserService +import com.webank.wedatasphere.dss.standard.app.sso.user.impl.SSOUserServiceImpl +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance + +import scala.collection.mutable + +/** + * HttpSSOIntegrationStandard 与 OriginSSOIntegrationStandard 的唯一区别,在于 SSORequestService 的不同。 + * + * 请注意,DSS 框架不推荐使用该一级规范实现类。 + * + * 1. OriginSSORequestServiceImpl:通常用于请求与 DSS 打通了一级规范的第三方系统,推荐使用; + * 2. HttpSSORequestServiceImpl:用于通过使用类似 token 的方式,访问没有打通一级规范的第三方系统,不推荐使用。 + * + * @author enjoyyin + * @since 1.1.0 + */ +class HttpSSOIntegrationStandard extends SSOIntegrationStandard { + + private val ssoPluginService: SSOPluginService = new OriginSSOPluginServiceImpl + private val ssoRequestService: SSORequestService = createSSORequestService() + private val ssoUserServices = new mutable.ArrayBuffer[SSOUserService]() + + protected def createSSORequestService(): SSORequestService = new HttpSSORequestServiceImpl + + override def getSSORequestService: SSORequestService = ssoRequestService + + override def getSSOPluginService: SSOPluginService = ssoPluginService + + override def getSSOUserService(appInstance: AppInstance): SSOUserService = + ssoUserServices.find(_.getAppInstance == appInstance).getOrElse(ssoUserServices synchronized { + ssoUserServices.find(_.getAppInstance == appInstance).getOrElse { + val service = new SSOUserServiceImpl + service.setAppInstance(appInstance) + ssoUserServices += service + service + } + }) + + override def init(): Unit = { + ssoPluginService.setSSOBuilderService(getSSOBuilderService) + } + + override def close(): Unit = HttpClient.close() + +} diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/OriginSSOIntegrationStandard.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/OriginSSOIntegrationStandard.scala new file mode 100644 index 000000000..748812a56 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/OriginSSOIntegrationStandard.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin + + +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.OriginSSORequestServiceImpl +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService +import com.webank.wedatasphere.dss.standard.app.sso.{SSOIntegrationStandard, SSOIntegrationStandardFactory} + +/** + * OriginSSOIntegrationStandard 与 HttpSSOIntegrationStandard 的唯一区别,在于 SSORequestService 的不同。 + * + * 请注意,DSS 框架强烈建议集成的第三方 AppConn 打通一级规范,并强烈推荐使用本一级规范实现类。 + * + * 1. OriginSSORequestServiceImpl:通常用于请求与 DSS 打通了一级规范的第三方系统,推荐使用; + * 2. HttpSSORequestServiceImpl:用于通过使用类似 token 的方式,访问没有打通一级规范的第三方系统,不推荐使用。 + * + */ +class OriginSSOIntegrationStandard private[origin]() extends HttpSSOIntegrationStandard { + + override protected def createSSORequestService(): SSORequestService = new OriginSSORequestServiceImpl + +} + +/** + * 默认的 SSO 规范工厂,通过该工厂可获取默认的 sso 集成规范 OriginSSOIntegrationStandard。 + *
+ * 为 DSS 框架强烈推荐使用的 一级规范 实现类,非特殊原因,如第三方系统为 Python 系统,用户无法对接 DSS 一级规范, + * 只能通过 token 的方式访问第三方系统,否则不推荐用户重写 SSOIntegrationStandardFactory。 + * 如何重写?请实现一个 {@code SSOIntegrationStandardFactory} 的实现类即可,DSS 框架会自动找到这个实现类, + * 并加载您指定的 {@code SSOIntegrationStandard}(推荐直接返回 {@code HttpSSOIntegrationStandard}) + * + */ +class OriginSSOIntegrationStandardFactory extends SSOIntegrationStandardFactory { + + private val ssoIntegrationStandard = new OriginSSOIntegrationStandard + + override def init(): Unit = { + ssoIntegrationStandard.init() + } + + override def getSSOIntegrationStandard: SSOIntegrationStandard = ssoIntegrationStandard +} \ No newline at end of file diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/client/HttpClient.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/client/HttpClient.scala new file mode 100644 index 000000000..ba92264c1 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/client/HttpClient.scala @@ -0,0 +1,114 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.client + +import java.net.URI +import java.text.SimpleDateFormat +import java.util +import java.util.Date + +import com.fasterxml.jackson.databind.ObjectMapper +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.httpclient.Client +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder +import org.apache.linkis.httpclient.request.HttpAction +import org.apache.commons.io.IOUtils +import org.apache.http.impl.cookie.BasicClientCookie + +import scala.collection.JavaConversions._ + + +object HttpClient extends Logging { + + private val dssClients = new util.HashMap[String, DWSHttpClient] + private val httpClients = new util.HashMap[String, Client] + + + + def getBaseUrl(url: String): String = { + val uri = new URI(url) + if (uri.getPort > 0){ + uri.getScheme + "://" + uri.getHost + ":" + uri.getPort + } else { + uri.getScheme + "://" + uri.getHost + } + } + + private def getClient[T](url: String, cacheMap: util.HashMap[String, T], createNewClient: String => T): T = { + val baseUrl = getBaseUrl(url) + if(!cacheMap.containsKey(baseUrl)) baseUrl.intern synchronized { + if(!cacheMap.containsKey(baseUrl)) { + info("create a new Client for url " + baseUrl) + val client = createNewClient(baseUrl) + cacheMap.put(baseUrl, client) + } + } + cacheMap.get(baseUrl) + } + + def getHttpClient(url: String, appName: String): Client = { + val baseUrl = getBaseUrl(url) + val clientConfig = DWSClientConfigBuilder. + newBuilder(). + addServerUrl(baseUrl). + connectionTimeout(connectTimeout). + setDWSVersion("v1"). + discoveryEnabled(false). + maxConnectionSize(maxConnection). + readTimeout(readTimeout).build() + new DWSHttpClient(clientConfig, appName + "-SSO-Client") + } + + def getDSSClient(dssUrl: String): DWSHttpClient = getClient(dssUrl, dssClients, baseUrl => { + val clientConfig = DWSClientConfigBuilder.newBuilder().setDWSVersion(dssVersion) + .addServerUrl(baseUrl).connectionTimeout(connectTimeout).discoveryEnabled(true) + .maxConnectionSize(maxConnection).readTimeout(readTimeout).build() + new DWSHttpClient(clientConfig, "DSS-Integration-Standard-Client") + }) + + def addCookies(dssMsg: DSSMsg, action: HttpAction): Unit = + dssMsg.getCookies.foreach { case (key, value) => + val basicClientCookie = new BasicClientCookie(key, value) + val domain = Utils.tryCatch(new URI(dssMsg.getDSSUrl).getHost)(_ => HttpClient.getBaseUrl(dssMsg.getDSSUrl)) + basicClientCookie.setDomain(domain) + basicClientCookie.setPath("/") + basicClientCookie.setExpiryDate(new Date(System.currentTimeMillis() + 1000 * 60 * 60 * 24 * 30L)) + info("Add cookie for get user info "+basicClientCookie.toString) + action.addCookie(basicClientCookie) + } + + def close(): Unit = { + dssClients.values().foreach(IOUtils.closeQuietly) + httpClients.values().foreach(IOUtils.closeQuietly) + } + + private var dssVersion = "v1" + private var maxConnection = 50 + private var connectTimeout = 300000 + private var readTimeout = 300000 + + //TODO 切换为linkis-common的JsonUtils + val objectMapper = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) + + def setDSSVersion(version: String): Unit = this.dssVersion = version + def setMaxConnection(maxConnection: Int): Unit = this.maxConnection = maxConnection + def setConnectTimeout(connectTimeout: Int): Unit = this.connectTimeout = connectTimeout + def setReadTimeout(readTimeout: Int): Unit = this.readTimeout = readTimeout + +} diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/conf/OriginSSOIntegrationConfiguration.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/conf/OriginSSOIntegrationConfiguration.scala new file mode 100644 index 000000000..09409f7cf --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/conf/OriginSSOIntegrationConfiguration.scala @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.standard.app.sso.origin.conf + +import org.apache.linkis.common.conf.CommonVars + + +object OriginSSOIntegrationConfiguration { + val SSO_MAX_HTTP_CONNECT_TIMEOUT = CommonVars("wds.dss.sso.httpclient.max.connectTimeout", 3000000) + val SSO_MAX_HTTP_READ_TIMEOUT = CommonVars("wds.dss.sso.httpclient.max.readTimeout", 3000000) +} diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOMsgParseOperation.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOMsgParseOperation.scala new file mode 100644 index 000000000..7597e8a39 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOMsgParseOperation.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.plugin + +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient +import com.webank.wedatasphere.dss.standard.app.sso.plugin.AbstractSSOMsgParseOperation +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.httpclient.Client + + +class OriginSSOMsgParseOperation extends AbstractSSOMsgParseOperation { + + override protected def getUser(dssMsg: DSSMsg): String = { + val dssUrl = dssMsg.getDSSUrl + val dwsHttpClient:Client=null + Utils.tryFinally({ + val dwsHttpClient = HttpClient.getHttpClient(dssUrl, "DSS") + val userInfoAction = new UserInfoAction + HttpClient.addCookies(dssMsg, userInfoAction) + dwsHttpClient.execute(userInfoAction) match { + case userInfoResult: UserInfoResult => + userInfoResult.getUserName + } + })(Utils.tryQuietly(dwsHttpClient.close())) + } + +} diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOPluginFilter.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOPluginFilter.scala new file mode 100644 index 000000000..f4979db57 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOPluginFilter.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.plugin + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard +import com.webank.wedatasphere.dss.standard.app.sso.origin.OriginSSOIntegrationStandardFactory +import com.webank.wedatasphere.dss.standard.app.sso.plugin.filter.SSOPluginFilter +import org.apache.linkis.common.utils.Logging + + +abstract class OriginSSOPluginFilter extends SSOPluginFilter with Logging{ + + private val factory = new OriginSSOIntegrationStandardFactory + + override def info(str: String): Unit = logger.info(str) + + override protected def getSSOIntegrationStandard: SSOIntegrationStandard = factory.getSSOIntegrationStandard +} diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOPluginServiceImpl.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOPluginServiceImpl.scala new file mode 100644 index 000000000..84f4c76d1 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginSSOPluginServiceImpl.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.plugin + +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOBuilderService +import com.webank.wedatasphere.dss.standard.app.sso.plugin.{SSOMsgParseOperation, SSOPluginService, WorkspacePlugin} +import com.webank.wedatasphere.dss.standard.common.service.AppServiceImpl +import org.apache.linkis.common.utils.Logging + + +class OriginSSOPluginServiceImpl extends AppServiceImpl with SSOPluginService with Logging { + + private var ssoBuilderService: SSOBuilderService = _ + private val ssoMsgParseOperation = new OriginSSOMsgParseOperation + private val workspacePlugin = new OriginWorkspacePlugin + workspacePlugin.setDssMsgCacheOperation(createDssMsgCacheOperation()) + + override def setSSOBuilderService(ssoBuilderService: SSOBuilderService): Unit = { + this.ssoBuilderService = ssoBuilderService + ssoMsgParseOperation.setSSOBuilderService(ssoBuilderService) + + } + + override def createSSOMsgParseOperation(): SSOMsgParseOperation = ssoMsgParseOperation + + override def close(): Unit = {} + + override def createWorkspacePluginOperation(): WorkspacePlugin = workspacePlugin +} \ No newline at end of file diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginWorkspacePlugin.java b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginWorkspacePlugin.java new file mode 100644 index 000000000..4313cdcda --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/OriginWorkspacePlugin.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.plugin; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.plugin.AbstractWorkspacePlugin; +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient; +import org.apache.linkis.httpclient.dws.DWSHttpClient; +import java.util.List; + + +public class OriginWorkspacePlugin extends AbstractWorkspacePlugin { + + @Override + protected List getAllUsers(DssMsgBuilderOperation.DSSMsg dssMsg) { + String dssUrl = dssMsg.getDSSUrl(); + DWSHttpClient dwsHttpClient = HttpClient.getDSSClient(dssUrl); + WorkspaceUsersAction workspaceUsersAction = new WorkspaceUsersAction(); + HttpClient.addCookies(dssMsg, workspaceUsersAction); + WorkspaceUsersResult result = (WorkspaceUsersResult) dwsHttpClient.execute(workspaceUsersAction); + return result.getUsers(); + } +} diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/UserInfoAction.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/UserInfoAction.scala new file mode 100644 index 000000000..01be79756 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/UserInfoAction.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.plugin + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.GetAction +import org.apache.linkis.httpclient.request.POSTAction + + +class UserInfoAction extends GetAction with DWSHttpAction { + override def suffixURLs: Array[String] = Array("user", "userInfo") +} +class WorkspaceUsersAction extends POSTAction with DWSHttpAction { + override def getRequestPayload: String = "" + + override def suffixURLs: Array[String] = Array("dss", "getUsersOfWorkspace") + + def setWorkspace(workspace: String): Unit = addRequestPayload("workspaceName", workspace) +} \ No newline at end of file diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/UserInfoResult.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/UserInfoResult.scala new file mode 100644 index 000000000..834b3e03f --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/plugin/UserInfoResult.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.plugin + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/user/userInfo") +class UserInfoResult extends DWSResult { + + @BeanProperty var userName: String = _ + +} + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/getUsersOfWorkspace") +class WorkspaceUsersResult extends DWSResult { + + @BeanProperty var users: java.util.List[String] = _ + +} \ No newline at end of file diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/HttpSSORequestOperation.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/HttpSSORequestOperation.scala new file mode 100644 index 000000000..a665c49c9 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/HttpSSORequestOperation.scala @@ -0,0 +1,103 @@ +package com.webank.wedatasphere.dss.standard.app.sso.origin.request + +import java.util +import java.util.concurrent.{ConcurrentHashMap, TimeUnit} + +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSHttpAction +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException +import com.webank.wedatasphere.dss.standard.app.sso.origin.conf.OriginSSOIntegrationConfiguration._ +import org.apache.commons.io.IOUtils +import org.apache.linkis.common.utils.{ByteTimeUtils, Logging, Utils} +import org.apache.linkis.httpclient.Client +import org.apache.linkis.httpclient.response.impl.DefaultHttpResult +import org.apache.linkis.httpclient.response.{HttpResult, Result} + +/** + * 用于通过使用类似 token 的方式,访问没有打通一级规范的第三方系统,不推荐使用。 + * @author enjoyyin + * @since 1.1.0 + */ +class HttpSSORequestOperation(private val appName: String) extends SSORequestOperation[DSSHttpAction, HttpResult] with Logging { + + /** + * 用于通过使用类似 token 的方式,访问没有打通一级规范的第三方系统,不推荐使用。 + * 为了能正常访问 第三方系统,请保证 req 已在 header 或 cookies 中注入了 token。 + * @param urlBuilder 由于第三方系统没有打通一级规范,所以 urlBuilder 一般为空 + * @param req Http 请求实体 + * @return Http 请求结果 + */ + override def requestWithSSO(urlBuilder: SSOUrlBuilderOperation, req: DSSHttpAction): HttpResult = { + val key = getKey(urlBuilder, req) + val url = getUrl(urlBuilder, req) + val httpClient = HttpSSORequestOperation.getHttpClient(appName, key, url) + httpClient.execute(req) match { + case result: HttpResult => result + case result => if (Result.isSuccessResult(result)) { + val defaultHttpResult = new DefaultHttpResult + defaultHttpResult.set(null, 200, url, null) + defaultHttpResult + } else throw new AppStandardErrorException(20300, s"Not support Result => ${result.getClass.getName}.") + } + } + + protected def getKey(urlBuilder: SSOUrlBuilderOperation, req: DSSHttpAction): String = { + val baseUrl = HttpClient.getBaseUrl(getUrl(urlBuilder, req)) + appName + req.getUser + baseUrl + } + + protected def getUrl(urlBuilder: SSOUrlBuilderOperation, req: DSSHttpAction): String = + if(urlBuilder == null) { + req.getURL + } else { + urlBuilder.getBuiltUrl + } + +} +object HttpSSORequestOperation extends Logging { + + val MAX_ACTIVE_TIME = ByteTimeUtils.timeStringAsMs("35m") + + private val httpClientLastAccessMap = new ConcurrentHashMap[String, Long] + private val httpClientMap = new util.HashMap[String, Client] + + Utils.defaultScheduler.scheduleWithFixedDelay(new Runnable { + override def run(): Unit = httpClientLastAccessMap.keySet().toArray.foreach { + case key: String => + if (System.currentTimeMillis - httpClientLastAccessMap.get(key) >= MAX_ACTIVE_TIME) httpClientMap synchronized { + if (httpClientLastAccessMap.containsKey(key)) { + httpClientLastAccessMap.remove(key) + IOUtils.closeQuietly(httpClientMap.get(key)) + httpClientMap.remove(key) + info(s"SSORequestOperation removed expired key($key).") + } + } + } + }, MAX_ACTIVE_TIME, MAX_ACTIVE_TIME, TimeUnit.MILLISECONDS) + + def getHttpClient(appName: String, key: String, url: String): Client = { + if (httpClientMap.containsKey(key) && System.currentTimeMillis - httpClientLastAccessMap.get(key) < MAX_ACTIVE_TIME) { + httpClientLastAccessMap.put(key, System.currentTimeMillis) + httpClientMap.get(key) + } else httpClientMap synchronized { + if (httpClientMap.containsKey(key) && System.currentTimeMillis - httpClientLastAccessMap.get(key) < MAX_ACTIVE_TIME) { + httpClientLastAccessMap.put(key, System.currentTimeMillis) + return httpClientMap.get(key) + } else { + if (httpClientMap.containsKey(key)) { + IOUtils.closeQuietly(httpClientMap.get(key)) + } + info(s"The appName $appName try to create http client with key $key.") + HttpClient.setConnectTimeout(SSO_MAX_HTTP_CONNECT_TIMEOUT.getValue) + HttpClient.setReadTimeout(SSO_MAX_HTTP_READ_TIMEOUT.getValue) + val httpClient = HttpClient.getHttpClient(url, appName) + httpClientMap.put(key, httpClient) + httpClientLastAccessMap.put(key, System.currentTimeMillis) + info(s"SSORequestOperation created a new http client for appName $appName with key($key).") + return httpClient + } + } + } +} \ No newline at end of file diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/HttpSSORequestServiceImpl.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/HttpSSORequestServiceImpl.scala new file mode 100644 index 000000000..7e53cf6ec --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/HttpSSORequestServiceImpl.scala @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.standard.app.sso.origin.request + +import java.util + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService +import com.webank.wedatasphere.dss.standard.common.service.AppServiceImpl + +/** + * 用于通过使用类似 token 的方式,访问没有打通一级规范的第三方系统,不推荐使用。 + * 如第三方系统为 Python 系统,用户无法对接 DSS 一级规范,只能通过 token 的方式访问第三方系统,则推荐使用该实现类。 + * 如何使用该类?请实现一个 {@code SSOIntegrationStandardFactory} 的实现类即可,DSS 框架会自动找到这个实现类, + * 并加载您的 {@code SSOIntegrationStandard}(推荐直接返回 {@code HttpSSOIntegrationStandard}) + * + * @author enjoyyin + * @since 1.1.0 + */ +class HttpSSORequestServiceImpl extends AppServiceImpl with SSORequestService { + + private val ssoRequestServices = new util.HashMap[String, HttpSSORequestOperation] + + protected def createHttpSSORequestOperation(appName: String): HttpSSORequestOperation = + new HttpSSORequestOperation(appName) + + override def createSSORequestOperation(appName: String): HttpSSORequestOperation = { + if(!ssoRequestServices.containsKey(appName)) synchronized { + if(!ssoRequestServices.containsKey(appName)) ssoRequestServices.put(appName, createHttpSSORequestOperation(appName)) + } + ssoRequestServices.get(appName) + } + +} diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/OriginSSORequestOperation.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/OriginSSORequestOperation.scala new file mode 100644 index 000000000..f2231da13 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/OriginSSORequestOperation.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.request + +import java.net.URI +import java.util.Date + +import com.webank.wedatasphere.dss.common.conf.DSSCommonConf +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation +import com.webank.wedatasphere.dss.standard.app.sso.builder.impl.SSOUrlBuilderOperationImpl +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient +import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSHttpAction +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException +import org.apache.http.impl.cookie.BasicClientCookie +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.httpclient.response.HttpResult + +import scala.collection.convert.wrapAsScala._ + +/** + * 用于请求与 DSS 打通了一级规范的第三方系统 + * @param appName AppConn 名称 + */ +class OriginSSORequestOperation private[request](appName: String) extends HttpSSORequestOperation(appName) { + + /** + * 用于请求与 DSS 打通了一级规范的第三方系统,也可以是请求 DSS 内嵌的数据应用工具。 + * @param urlBuilder 不能为空,且必须为 SSOUrlBuilderOperationImpl 的实现类 + * @param req DSSHttpAction 实现类 + * @return HTTP 请求的结果 + */ + override def requestWithSSO(urlBuilder: SSOUrlBuilderOperation, req: DSSHttpAction): HttpResult = { + urlBuilder match { + case urlBuilderOperationImpl: SSOUrlBuilderOperationImpl => + val cookies = urlBuilderOperationImpl.getCookies + cookies.foreach { + case (key, value) => + val basicClientCookie = new BasicClientCookie(key, value) + val domain = Utils.tryCatch(new URI(urlBuilder.getBuiltUrl).getHost)(_ => "") + basicClientCookie.setDomain(domain) + basicClientCookie.setPath("/") + basicClientCookie.setExpiryDate(new Date(System.currentTimeMillis() + 1000 * 60 * 60 * 24 * 30L)) + info("Add cookie for get user info " + basicClientCookie.toString) + req.addCookie(basicClientCookie) + } + case _ if urlBuilder != null => + throw new AppStandardErrorException(20300, s"Not support SSOUrlBuilderOperation => ${urlBuilder.getClass.getName}.") + case _ => + throw new AppStandardErrorException(20300, "SSOUrlBuilderOperation is null.") + } + super.requestWithSSO(urlBuilder, req) + } + + override protected def getKey(urlBuilder: SSOUrlBuilderOperation, req: DSSHttpAction): String = { + val baseUrl = HttpClient.getBaseUrl(urlBuilder.getBuiltUrl) + urlBuilder match { + case builder: SSOUrlBuilderOperationImpl => + builder.getCookies.find(_._1 == DSSCommonConf.DSS_TOKEN_TICKET_KEY.getValue).foreach{ case (_, ticketId) => + return appName + ticketId + baseUrl + } + } + throw new AppStandardErrorException(20300, "User has not login, please login first.") + } +} \ No newline at end of file diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/OriginSSORequestServiceImpl.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/OriginSSORequestServiceImpl.scala new file mode 100644 index 000000000..fa76bd584 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/OriginSSORequestServiceImpl.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.request + +/** + * 通常用于请求与 DSS 打通了一级规范的第三方系统,是默认的使用方案 + */ +class OriginSSORequestServiceImpl extends HttpSSORequestServiceImpl { + + override protected def createHttpSSORequestOperation(appName: String): HttpSSORequestOperation = + new OriginSSORequestOperation(appName) + + override def createSSORequestOperation(appName: String): OriginSSORequestOperation = { + super.createSSORequestOperation(appName).asInstanceOf[OriginSSORequestOperation] + } +} diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/action/DSSHttpAction.scala b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/action/DSSHttpAction.scala new file mode 100644 index 000000000..167de8ee7 --- /dev/null +++ b/dss-standard/sso-standard/origin-sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/origin/request/action/DSSHttpAction.scala @@ -0,0 +1,98 @@ +package com.webank.wedatasphere.dss.standard.app.sso.origin.request.action + +import java.io.{Closeable, InputStream} +import java.util + +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils +import org.apache.commons.io.IOUtils +import org.apache.http.HttpResponse +import org.apache.http.client.methods.CloseableHttpResponse +import org.apache.linkis.httpclient.request._ + +/** + * + * @date 2022-03-16 + * @author enjoyyin + * @since 1.1.0 + */ +trait DSSHttpAction extends HttpAction with UserAction { + + private var url: String = _ + private var user: String = _ + + override def getURL: String = url + + def setUrl(url: String): Unit = { + this.url = url + } + + override def setUser(user: String): Unit = { + this.user = user + } + + override def getUser: String = user + +} + +class DSSGetAction extends GetAction with DSSHttpAction + +class DSSPostAction extends POSTAction with DSSHttpAction { + override def getRequestPayload: String = if (getRequestPayloads.isEmpty) "" else DSSCommonUtils.COMMON_GSON.toJson(getRequestPayloads) +} + +class DSSDeleteAction extends DeleteAction with DSSHttpAction + +class DSSPutAction extends PutAction with DSSHttpAction { + override def getRequestPayload: String = if (getRequestPayloads.isEmpty) "" else DSSCommonUtils.COMMON_GSON.toJson(getRequestPayloads) +} + +class DSSDownloadAction extends DSSGetAction with DownloadAction with DSSHttpAction with Closeable { + + private var inputStream: InputStream = _ + private var response: HttpResponse = _ + + def getInputStream: InputStream = inputStream + + override def write(inputStream: InputStream): Unit = { + this.inputStream = inputStream + } + + override def getResponse: HttpResponse = response + + override def setResponse(response: HttpResponse): Unit = this.response = response + + override def close(): Unit = { + if(inputStream != null) { + IOUtils.closeQuietly(inputStream) + } + response match { + case r: CloseableHttpResponse => + IOUtils.closeQuietly(r) + case _ => + } + } +} + +class DSSUploadAction(override val files: util.Map[String, String]) + extends DSSPostAction with UploadAction { + + private var _binaryBodies: util.List[BinaryBody] = _ + private var _inputStreams: util.Map[String, java.io.InputStream] = _ + + def this() = this(new util.HashMap[String, String]) + + def this(binaryBodies: util.List[BinaryBody]) = { + this(new util.HashMap[String, String]) + _binaryBodies = binaryBodies + } + + def setBinaryBodies(binaryBodies: util.List[BinaryBody]): Unit = _binaryBodies = binaryBodies + + def setInputStreams(inputStreams: util.Map[String, java.io.InputStream]): Unit = + _inputStreams = inputStreams + + override def inputStreams: util.Map[String, java.io.InputStream] = _inputStreams + + override def binaryBodies: util.List[BinaryBody] = _binaryBodies + +} \ No newline at end of file diff --git a/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml b/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml new file mode 100644 index 000000000..e6e31322d --- /dev/null +++ b/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml @@ -0,0 +1,89 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + spring-origin-sso-integration-plugin + + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + com.webank.wedatasphere.dss + dss-origin-sso-integration-standard + ${dss.version} + + + org.springframework + spring-core + ${spring.version} + compile + + + org.springframework + spring-context + ${spring.version} + compile + + + org.springframework + spring-web + ${spring.version} + compile + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + + + \ No newline at end of file diff --git a/dss-standard/sso-standard/spring-origin-sso-integration-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/origin/filter/spring/SpringOriginSSOPluginFilter.java b/dss-standard/sso-standard/spring-origin-sso-integration-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/origin/filter/spring/SpringOriginSSOPluginFilter.java new file mode 100644 index 000000000..3c69165a5 --- /dev/null +++ b/dss-standard/sso-standard/spring-origin-sso-integration-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/origin/filter/spring/SpringOriginSSOPluginFilter.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.origin.filter.spring; + +import com.webank.wedatasphere.dss.standard.app.sso.origin.plugin.OriginSSOPluginFilter; +import com.webank.wedatasphere.dss.standard.app.sso.plugin.filter.UserInterceptor; + +import javax.servlet.FilterConfig; +import org.springframework.stereotype.Component; +import org.springframework.web.context.WebApplicationContext; +import org.springframework.web.context.support.WebApplicationContextUtils; + + +@Component +public class SpringOriginSSOPluginFilter extends OriginSSOPluginFilter { + + @Override + public UserInterceptor getUserInterceptor(FilterConfig filterConfig) { + WebApplicationContext webApplicationContext = + WebApplicationContextUtils.getRequiredWebApplicationContext(filterConfig.getServletContext()); + return webApplicationContext.getBean(UserInterceptor.class); + } +} diff --git a/dss-standard/sso-standard/sso-integration-standard/pom.xml b/dss-standard/sso-standard/sso-integration-standard/pom.xml new file mode 100644 index 000000000..139c364d9 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/pom.xml @@ -0,0 +1,83 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-sso-integration-standard + + + + javax.servlet + javax.servlet-api + 3.1.0 + + + com.webank.wedatasphere.dss + dss-standard-common + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + scala-compile-first + + + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + + + \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/SSOIntegrationStandard.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/SSOIntegrationStandard.java new file mode 100644 index 000000000..06649de45 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/SSOIntegrationStandard.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOBuilderService; +import com.webank.wedatasphere.dss.standard.app.sso.builder.impl.SSOBuilderServiceImplImpl; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.app.sso.plugin.SSOPluginService; +import com.webank.wedatasphere.dss.standard.app.sso.user.SSOUserService; +import com.webank.wedatasphere.dss.standard.common.core.AppStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +/** + * DSS 与 第三方系统的 SSO 免登录跳转规范,已提供默认实现,用户无需实现该 AppConn 的任何方法。 + */ +public interface SSOIntegrationStandard extends AppStandard { + + default SSOBuilderService getSSOBuilderService() { + return SSOBuilderServiceImplImpl.getSSOBuilderService(); + } + + /** + * 提供通用的、可以向与 DSS 集成的第三方 AppConn 系统发送前端或后台请求的服务能力。 + * @return SSORequestService 实现类 + */ + SSORequestService getSSORequestService(); + + SSOPluginService getSSOPluginService(); + + /** + * DSS 用户与第三方 AppConn 的用户同步服务 + * @return SSOUserService 实现类 + */ + SSOUserService getSSOUserService(AppInstance appInstance); + + @Override + default String getStandardName() { + return "ssoIntegrationStandard"; + } + + @Override + default int getGrade() { + return 1; + } + + @Override + default boolean isNecessary() { + return true; + } +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/SSOIntegrationStandardFactory.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/SSOIntegrationStandardFactory.java new file mode 100644 index 000000000..e2cb059da --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/SSOIntegrationStandardFactory.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso; + +import java.io.Serializable; + +/** + * DSS SSO 一级规范工厂,通过该工厂可获取 DSS 的一级 SSO 免密互通规范,{@code AbstractOnlySSOAppConn} 会使用该工厂 + * 来加载 {@code SSOIntegrationStandard}。 + * 已提供了默认的实现类 {@code OriginSSOIntegrationStandardFactory},为 DSS 默认提供、且强烈推荐使用的实现类。 + *
+ * 如有特殊原因,如第三方系统为 Python 系统,用户无法对接 DSS 一级规范, + * 只能通过 token 的方式访问第三方系统,则推荐用户重写 SSOIntegrationStandardFactory。 + * 如何重写?请实现一个 {@code SSOIntegrationStandardFactory} 的实现类即可,DSS 框架会自动找到这个实现类, + * 并加载您指定的 {@code SSOIntegrationStandard}(推荐直接返回 {@code HttpSSOIntegrationStandard})。 + */ +public interface SSOIntegrationStandardFactory extends Serializable { + + void init(); + + SSOIntegrationStandard getSSOIntegrationStandard(); + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/Workspace.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/Workspace.java new file mode 100644 index 000000000..7a8eb02bd --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/Workspace.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso; + +import com.webank.wedatasphere.dss.common.entity.DSSWorkspace; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import org.apache.linkis.common.conf.Configuration; + +import java.util.HashMap; +import java.util.Map; + + +public class Workspace implements DSSWorkspace { + + protected long workspaceId; + protected String workspaceName; + protected Map cookies = new HashMap<>(); + protected String dssUrl = Configuration.GATEWAY_URL().getValue(); + + @Override + public long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(long workspaceId) { + this.workspaceId = workspaceId; + } + + @Override + public String getWorkspaceName() { + return this.workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public Map getCookies() { + return cookies; + } + + public void addCookie(String key, String value) { + cookies.put(key, value); + } + + public void setCookies(Map cookies) { + this.cookies = cookies; + } + + public String getDssUrl() { + return dssUrl; + } + + public void setDssUrl(String dssUrl) { + this.dssUrl = dssUrl; + } + + @Override + public String toString() { + return "Workspace{" + + "workspaceId=" + workspaceId + + ", workspaceName='" + workspaceName + '\'' + + ", cookies=" + cookies + + ", dssUrl='" + dssUrl + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/DssMsgBuilderOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/DssMsgBuilderOperation.java new file mode 100644 index 000000000..570d6bd81 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/DssMsgBuilderOperation.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.builder; + +import com.webank.wedatasphere.dss.standard.common.service.Operation; + +import java.util.Map; + + +public interface DssMsgBuilderOperation extends Operation { + + DssMsgBuilderOperation setQueryString(String queryString); + + DssMsgBuilderOperation setParameterMap(Map parameterMap); + + boolean isDSSMsgRequest(); + + DSSMsg getBuiltMsg(); + + interface DSSMsg { + + String getRedirectUrl(); + + String getWorkspaceName(); + + String getDSSUrl(); + + String getAppName(); + + Map getCookies(); + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/SSOBuilderService.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/SSOBuilderService.java new file mode 100644 index 000000000..0b9268b02 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/SSOBuilderService.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.builder; + + +import com.webank.wedatasphere.dss.standard.common.service.AppService; + + +public interface SSOBuilderService extends AppService { + + SSOUrlBuilderOperation createSSOUrlBuilderOperation(); + + DssMsgBuilderOperation createDssMsgBuilderOperation(); + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/SSOUrlBuilderOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/SSOUrlBuilderOperation.java new file mode 100644 index 000000000..8323c1de3 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/SSOUrlBuilderOperation.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.builder; + + +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; +import com.webank.wedatasphere.dss.standard.common.service.Operation; + + +public interface SSOUrlBuilderOperation extends Operation { + + SSOUrlBuilderOperation setReqUrl(String reqUrl); + + SSOUrlBuilderOperation setWorkspace(String workspaceName); + + SSOUrlBuilderOperation setDSSUrl(String dssUrl); + + SSOUrlBuilderOperation setAppName(String appConnName); + + SSOUrlBuilderOperation addCookie(String key, String value); + + SSOUrlBuilderOperation redirectTo(String redirectUrl); + + SSOUrlBuilderOperation addQueryParameter(String key, String value); + + SSOUrlBuilderOperation copy(); + + String getBuiltUrl() throws AppStandardErrorException; +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/DSSMsgImpl.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/DSSMsgImpl.java new file mode 100644 index 000000000..b7ccf84fb --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/DSSMsgImpl.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.builder.impl; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg; +import java.util.Map; + + +public class DSSMsgImpl implements DSSMsg { + + private String redirectUrl; + private String workspaceName; + private String dssUrl; + private String appName; + private Map cookies; + + @Override + public String getRedirectUrl() { + return redirectUrl; + } + + @Override + public String getWorkspaceName() { + return workspaceName; + } + + @Override + public String getDSSUrl() { + return dssUrl; + } + + @Override + public Map getCookies() { + return cookies; + } + + public void setRedirectUrl(String redirectUrl) { + this.redirectUrl = redirectUrl; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public void setDSSUrl(String dssUrl) { + this.dssUrl = dssUrl; + } + + @Override + public String getAppName() { + return appName; + } + + public void setAppName(String appName) { + this.appName = appName; + } + + public void setCookies(Map cookies) { + this.cookies = cookies; + } +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/DssMsgBuilderOperationImpl.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/DssMsgBuilderOperationImpl.java new file mode 100644 index 000000000..c12cce70f --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/DssMsgBuilderOperationImpl.java @@ -0,0 +1,83 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.builder.impl; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + + +public class DssMsgBuilderOperationImpl implements DssMsgBuilderOperation { + + private static final Logger LOGGER = LoggerFactory.getLogger(DssMsgBuilderOperationImpl.class); + + private String queryString; + private Map parameterMap; + + @Override + public DssMsgBuilderOperation setQueryString(String queryString) { + this.queryString = queryString; + return this; + } + + @Override + public DssMsgBuilderOperation setParameterMap(Map parameterMap) { + this.parameterMap = parameterMap; + return this; + } + + @Override + public boolean isDSSMsgRequest() { + return parameterMap.containsKey("dssurl"); + } + + private String getOrNull(String key) { + String[] values = parameterMap.get(key); + if(values == null || values.length == 0) { + return null; + } else { + return values[0]; + } + } + + @Override + public DSSMsg getBuiltMsg() { + DSSMsgImpl dssMsg = new DSSMsgImpl(); + dssMsg.setRedirectUrl(getOrNull("redirect")); + dssMsg.setDSSUrl(getOrNull("dssurl")); + dssMsg.setWorkspaceName(getOrNull("workspace")); + dssMsg.setAppName(getOrNull("appName")); + Map cookies = new HashMap<>(); + String cookiesStr = getOrNull("cookies"); + if(StringUtils.isNotBlank(cookiesStr)) { + Arrays.stream(cookiesStr.split(";")).forEach(cookie -> { + int index = cookie.indexOf('='); + String key = cookie.substring(0, index).trim(); + String value = cookie.substring(index + 1).trim(); + cookies.put(key, value); + }); + } + LOGGER.info("Set cookies from dssMsg: "+cookies.toString()); + dssMsg.setCookies(cookies); + return dssMsg; + } +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/SSOBuilderServiceImplImpl.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/SSOBuilderServiceImplImpl.java new file mode 100644 index 000000000..10ed2d681 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/SSOBuilderServiceImplImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.builder.impl; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOBuilderService; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.common.service.AppServiceImpl; + + +public class SSOBuilderServiceImplImpl extends AppServiceImpl implements SSOBuilderService { + + private static SSOBuilderService ssoBuilderService; + + public static SSOBuilderService getSSOBuilderService() { + if(ssoBuilderService == null) { + synchronized (SSOBuilderServiceImplImpl.class) { + if(ssoBuilderService == null) { + ssoBuilderService = new SSOBuilderServiceImplImpl(); + } + } + } + return ssoBuilderService; + } + + @Override + public SSOUrlBuilderOperation createSSOUrlBuilderOperation() { + return new SSOUrlBuilderOperationImpl(); + } + + @Override + public DssMsgBuilderOperation createDssMsgBuilderOperation() { + return new DssMsgBuilderOperationImpl(); + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/SSOUrlBuilderOperationImpl.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/SSOUrlBuilderOperationImpl.java new file mode 100644 index 000000000..ae562663f --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/builder/impl/SSOUrlBuilderOperationImpl.java @@ -0,0 +1,149 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.builder.impl; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; +import org.apache.commons.lang.StringUtils; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + + +public class SSOUrlBuilderOperationImpl implements SSOUrlBuilderOperation { + + private static final String SSO_URL_FORMAT = "%s?dssurl=%s&cookies=%s&workspace=%s&appName=%s"; + private static final String SSO_REDIRECT_URL_FORMAT = "%s?redirect=%s&dssurl=%s&cookies=%s&workspace=%s&appName=%s"; + private String workspaceName; + private Map cookies = new HashMap<>(); + private Map queryParameters = new HashMap<>(); + private String dssUrl; + private String redirectUrl; + private String reqUrl; + private String appName; + + protected String urlEncode(String str) throws AppStandardErrorException { + try { + return URLEncoder.encode(str, "utf-8"); + } catch (UnsupportedEncodingException e) { + throw new AppStandardErrorException(60001, "Encode string failed! string: " + str, e); + } + } + + @Override + public SSOUrlBuilderOperation setWorkspace(String workspaceName) { + this.workspaceName = workspaceName; + return this; + } + + @Override + public SSOUrlBuilderOperation addCookie(String key, String value) { + cookies.put(key, value); + return this; + } + + @Override + public SSOUrlBuilderOperation redirectTo(String redirectUrl) { + this.redirectUrl = redirectUrl; + return this; + } + + @Override + public SSOUrlBuilderOperation setReqUrl(String reqUrl) { + this.reqUrl = reqUrl; + return this; + } + + @Override + public SSOUrlBuilderOperation setDSSUrl(String dssUrl) { + this.dssUrl = dssUrl; + return this; + } + + @Override + public SSOUrlBuilderOperation setAppName(String appName) { + this.appName = appName; + return this; + } + + @Override + public SSOUrlBuilderOperation addQueryParameter(String key, String value) { + queryParameters.put(key, value); + return this; + } + + @Override + public String getBuiltUrl() throws AppStandardErrorException { + String cookieStr = cookies.entrySet().stream().map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining(";")); + String url; + if(StringUtils.isNotBlank(redirectUrl)) { + url = String.format(SSO_REDIRECT_URL_FORMAT, reqUrl, redirectUrl, urlEncode(dssUrl), + urlEncode(cookieStr), urlEncode(workspaceName), appName); + } else { + url = String.format(SSO_URL_FORMAT, reqUrl, urlEncode(dssUrl), + urlEncode(cookieStr), urlEncode(workspaceName), appName); + } + if(queryParameters.isEmpty()) { + return url; + } else { + String queryParameterStr = queryParameters.entrySet().stream().map(entry -> entry.getKey() + "=" + urlEncode(entry.getValue())) + .collect(Collectors.joining("&")); + return url + "&" + queryParameterStr; + } + } + + public String getWorkspaceName() { + return workspaceName; + } + + public Map getCookies() { + return cookies; + } + + public String getDssUrl() { + return dssUrl; + } + + public String getRedirectUrl() { + return redirectUrl; + } + + public String getReqUrl() { + return reqUrl; + } + + public String getAppName() { + return appName; + } + + @Override + public SSOUrlBuilderOperation copy() { + SSOUrlBuilderOperationImpl operation = new SSOUrlBuilderOperationImpl(); + operation.appName = appName; + operation.cookies = cookies; + operation.dssUrl = dssUrl; + operation.redirectUrl = redirectUrl; + operation.reqUrl = reqUrl; + operation.workspaceName = workspaceName; + return operation; + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/operation/AbstractOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/operation/AbstractOperation.java new file mode 100644 index 000000000..d9abd3617 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/operation/AbstractOperation.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.dss.standard.app.sso.operation; + +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppIntegrationService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.service.Operation; + +/** + * @author enjoyyin + * @date 2022-03-09 + * @since 0.5.0 + */ +public abstract class AbstractOperation + implements Operation { + + protected SSORequestOperation ssoRequestOperation; + protected AppIntegrationService service; + + /** + * This method is used to create a SSORequestOperation. + * If the third-part AppConn wants to use SSORequestOperation which dependents the HttpClient of Linkis + * to request the third-part system, please override this method. Otherwise, the operations of + * third-part AppConn has no necessity to override this method. + * @return the appConn name + */ + protected abstract String getAppConnName(); + + @Override + public void init() { + if(getAppConnName() != null) { + this.ssoRequestOperation = service.getSSORequestService().createSSORequestOperation(getAppConnName()); + } + } + + protected String getBaseUrl() { + return service.getAppInstance().getBaseUrl(); + } + + protected String mergeUrl(String url, String suffix) { + if(url.endsWith("/")) { + return url + suffix; + } else { + return url + "/" + suffix; + } + } + + protected String mergeBaseUrl(String suffix) { + return mergeUrl(getBaseUrl(), suffix); + } + + protected String toJson(Object object) { + if(object == null) { + return null; + } else { + return DSSCommonUtils.COMMON_GSON.toJson(object); + } + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/AbstractSSOMsgParseOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/AbstractSSOMsgParseOperation.java new file mode 100644 index 000000000..58776b418 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/AbstractSSOMsgParseOperation.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOBuilderService; +import com.webank.wedatasphere.dss.standard.app.sso.plugin.impl.SSOMsgImpl; +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg; +import javax.servlet.http.HttpServletRequest; + + +public abstract class AbstractSSOMsgParseOperation implements SSOMsgParseOperation { + + protected SSOBuilderService ssoBuilderService; + + public void setSSOBuilderService(SSOBuilderService ssoBuilderService) { + this.ssoBuilderService = ssoBuilderService; + } + + @Override + public boolean isDssRequest(HttpServletRequest request) { + return ssoBuilderService.createDssMsgBuilderOperation().setParameterMap(request.getParameterMap()).isDSSMsgRequest(); + } + + @Override + public DSSMsg getDSSMsg(HttpServletRequest request) { + return ssoBuilderService.createDssMsgBuilderOperation() + .setParameterMap(request.getParameterMap()).getBuiltMsg(); + } + + @Override + public SSOMsg getSSOMsg(HttpServletRequest request) { + DSSMsg dssMsg = getDSSMsg(request); + SSOMsg ssoMsg = createSSOMsg(); + String user = getUser(dssMsg); + setSSOMsg(ssoMsg, dssMsg, user); + return ssoMsg; + } + + protected SSOMsg createSSOMsg() { + return new SSOMsgImpl(); + } + + protected void setSSOMsg(SSOMsg ssoMsg, DSSMsg dssMsg, String user) { + SSOMsgImpl ssoMsgImpl = (SSOMsgImpl) ssoMsg; + ssoMsgImpl.setRedirectUrl(dssMsg.getRedirectUrl()); + ssoMsgImpl.setWorkspaceName(dssMsg.getWorkspaceName()); + ssoMsgImpl.setUser(user); + } + + protected abstract String getUser(DSSMsg dssMsg); +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/AbstractWorkspacePlugin.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/AbstractWorkspacePlugin.java new file mode 100644 index 000000000..f7ea6fee6 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/AbstractWorkspacePlugin.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg; +import java.util.List; +import javax.servlet.http.HttpServletRequest; + + +public abstract class AbstractWorkspacePlugin implements WorkspacePlugin { + + private DssMsgCacheOperation dssMsgCacheOperation; + + public void setDssMsgCacheOperation(DssMsgCacheOperation dssMsgCacheOperation) { + this.dssMsgCacheOperation = dssMsgCacheOperation; + } + + @Override + public String getWorkspaceName(HttpServletRequest req) { + return dssMsgCacheOperation.getWorkspaceInSession(req); + } + + @Override + public List getAllUsers(HttpServletRequest req) { + DSSMsg dssMsg = dssMsgCacheOperation.getDSSMsgInSession(req); + return getAllUsers(dssMsg); + } + + protected abstract List getAllUsers(DSSMsg dssMsg); +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/DssMsgCacheOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/DssMsgCacheOperation.java new file mode 100644 index 000000000..04fdf906f --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/DssMsgCacheOperation.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin; + +import com.webank.wedatasphere.dss.standard.common.service.Operation; +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg; +import javax.servlet.http.HttpServletRequest; + + +public interface DssMsgCacheOperation extends Operation { + + void setWorkspaceToSession(HttpServletRequest req, String workspaceName); + + String getWorkspaceInSession(HttpServletRequest req); + + DSSMsg getDSSMsgInSession(HttpServletRequest request); + + void setDSSMsgToSession(DSSMsg dssMsg, HttpServletRequest request); + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOMsg.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOMsg.java new file mode 100644 index 000000000..4b2dac42b --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOMsg.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin; + + +public interface SSOMsg { + + String getRedirectUrl(); + + String getWorkspaceName(); + + String getUser(); + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOMsgParseOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOMsgParseOperation.java new file mode 100644 index 000000000..26f68828f --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOMsgParseOperation.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin; + +import com.webank.wedatasphere.dss.standard.common.service.Operation; +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg; +import javax.servlet.http.HttpServletRequest; + +/** + * + */ +public interface SSOMsgParseOperation extends Operation { + + boolean isDssRequest(HttpServletRequest request); + + DSSMsg getDSSMsg(HttpServletRequest request); + + SSOMsg getSSOMsg(HttpServletRequest request); + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOPluginService.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOPluginService.java new file mode 100644 index 000000000..47b53856c --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOPluginService.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOBuilderService; +import com.webank.wedatasphere.dss.standard.common.service.AppService; +import java.io.Closeable; + +/** + * 与 `SSOBuilderService` 对应,第三方系统在引入 DSS SSO Jar 后, + * 该Jar 会使用该 Service 解析第三方系统的后台 `HttpServletRequest`,获取所需的 DSS 实体信息 + */ +public interface SSOPluginService extends AppService, Closeable { + + void setSSOBuilderService(SSOBuilderService ssoBuilderService); + + SSOMsgParseOperation createSSOMsgParseOperation(); + + WorkspacePlugin createWorkspacePluginOperation(); + + default DssMsgCacheOperation createDssMsgCacheOperation() { + return DssMsgCacheOperationImpl.getDssMsgCacheOperation(); + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/WorkspacePlugin.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/WorkspacePlugin.java new file mode 100644 index 000000000..844ff4274 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/WorkspacePlugin.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin; + +import com.webank.wedatasphere.dss.standard.common.service.Operation; + +import java.util.List; +import javax.servlet.http.HttpServletRequest; + + +public interface WorkspacePlugin extends Operation { + + List getAllUsers(HttpServletRequest req); + + String getWorkspaceName(HttpServletRequest req); + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/DSSInternalUserInterceptor.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/DSSInternalUserInterceptor.java new file mode 100644 index 000000000..8deca7f10 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/DSSInternalUserInterceptor.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin.filter; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg; +import javax.servlet.http.HttpServletRequest; + + +public interface DSSInternalUserInterceptor extends UserInterceptor { + + HttpServletRequest addCookiesToRequest(DSSMsg dssMsg, HttpServletRequest req); + +} \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/HttpRequestUserInterceptor.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/HttpRequestUserInterceptor.java new file mode 100644 index 000000000..cb1a8f235 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/HttpRequestUserInterceptor.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin.filter; + +import javax.servlet.http.HttpServletRequest; + + +public interface HttpRequestUserInterceptor extends UserInterceptor { + + HttpServletRequest addUserToRequest(String user, HttpServletRequest req); + +} \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/HttpSessionUserInterceptor.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/HttpSessionUserInterceptor.java new file mode 100644 index 000000000..fa7038531 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/HttpSessionUserInterceptor.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin.filter; + +import javax.servlet.http.HttpServletRequest; + + +public interface HttpSessionUserInterceptor extends UserInterceptor { + + void addUserToSession(String user, HttpServletRequest req); + +} \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/SSOPluginFilter.scala b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/SSOPluginFilter.scala new file mode 100644 index 000000000..7edc6d7c9 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/SSOPluginFilter.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin.filter + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard +import javax.servlet._ +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} +import org.apache.commons.lang.StringUtils + + +abstract class SSOPluginFilter extends Filter { + + private var userInterceptor: UserInterceptor = _ + + override def init(filterConfig: FilterConfig): Unit = { + userInterceptor = getUserInterceptor(filterConfig) + getSSOIntegrationStandard.init() + } + + def info(str: String): Unit + + protected def getUserInterceptor(filterConfig: FilterConfig): UserInterceptor + + protected def getSSOIntegrationStandard: SSOIntegrationStandard + + override def doFilter(servletRequest: ServletRequest, servletResponse: ServletResponse, + filterChain: FilterChain): Unit = { + val req = servletRequest.asInstanceOf[HttpServletRequest] + val resp = servletResponse.asInstanceOf[HttpServletResponse] + val ssoPluginService = getSSOIntegrationStandard.getSSOPluginService + val dssMsg = ssoPluginService.createSSOMsgParseOperation().getDSSMsg(req) + if (ssoPluginService.createSSOMsgParseOperation.isDssRequest(req)) { + val (redirectUrl, workspaceName, wrappedReq) = if(!userInterceptor.isUserExistInSession(req)) { + val ssoMsg = ssoPluginService.createSSOMsgParseOperation.getSSOMsg(req) + val username = ssoMsg.getUser + val wrappedReq = userInterceptor match { + case interceptor: HttpSessionUserInterceptor => + interceptor.addUserToSession(username, req) + req + case interceptor: HttpRequestUserInterceptor => interceptor.addUserToRequest(username, req) + case interceptor: DSSInternalUserInterceptor => interceptor.addCookiesToRequest(dssMsg, req) + } + if(wrappedReq != req) { + wrappedReq.getCookies.foreach(resp.addCookie) + } + info(s"DSS User: $username succeed to login. ") + (ssoMsg.getRedirectUrl, ssoMsg.getWorkspaceName, wrappedReq) + } else { + (dssMsg.getRedirectUrl, dssMsg.getWorkspaceName, req) + } + val workspaceOperation = ssoPluginService.createDssMsgCacheOperation() + if(workspaceName != workspaceOperation.getWorkspaceInSession(req)) { + info(s"Set DSS workspace to: $workspaceName.") + workspaceOperation.setWorkspaceToSession(req, workspaceName) + } + if(workspaceOperation.getDSSMsgInSession(req) == null) { + workspaceOperation.setDSSMsgToSession(dssMsg, req) + } + if(StringUtils.isNotBlank(redirectUrl)) resp.sendRedirect(redirectUrl) + else filterChain.doFilter(wrappedReq, servletResponse) + } else { + filterChain.doFilter(servletRequest, servletResponse) + } + } + + override def destroy(): Unit = {} + +} \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/UserInterceptor.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/UserInterceptor.java new file mode 100644 index 000000000..acf745682 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/filter/UserInterceptor.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin.filter; + +import javax.servlet.http.HttpServletRequest; + + +public interface UserInterceptor { + + boolean isUserExistInSession(HttpServletRequest req); + + String getUser(HttpServletRequest req); + +} \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/impl/SSOMsgImpl.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/impl/SSOMsgImpl.java new file mode 100644 index 000000000..05da2833a --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/plugin/impl/SSOMsgImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin.impl; + +import com.webank.wedatasphere.dss.standard.app.sso.plugin.SSOMsg; + + +public class SSOMsgImpl implements SSOMsg { + + private String redirectUrl; + private String user; + private String workspaceName; + + @Override + public String getRedirectUrl() { + return redirectUrl; + } + + public void setRedirectUrl(String redirectUrl) { + this.redirectUrl = redirectUrl; + } + + @Override + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + @Override + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/ref/WorkspaceRequestRef.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/ref/WorkspaceRequestRef.java new file mode 100644 index 000000000..fc9558c9e --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/ref/WorkspaceRequestRef.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.ref; + +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; + +public interface WorkspaceRequestRef extends RequestRef { + + default Workspace getWorkspace(){ + return null; + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/request/SSORequestOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/request/SSORequestOperation.java new file mode 100644 index 000000000..98fbd26da --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/request/SSORequestOperation.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.request; + +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; +import com.webank.wedatasphere.dss.standard.common.service.Operation; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; + +/** + * 向与 DSS 集成的第三方 AppConn 系统发送前端或后台请求。 + * @param + * @param + */ +public interface SSORequestOperation extends Operation { + + /** + * 提供通用的、可以向与 DSS 集成的第三方 AppConn 系统发送前端或后台请求的操作能力。 + * @param urlBuilder 向第三方 AppConn 系统发起请求的 URL 构造器 + * @param req Http 请求实体 + * @return Http 请求结果 + * @throws AppStandardErrorException Http 请求失败则抛出该异常 + */ + R requestWithSSO(SSOUrlBuilderOperation urlBuilder, T req) throws AppStandardErrorException; + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/request/SSORequestService.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/request/SSORequestService.java new file mode 100644 index 000000000..697c5784b --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/request/SSORequestService.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.request; + + +import com.webank.wedatasphere.dss.standard.common.service.AppService; + +/** + * 提供通用的、可以向与 DSS 集成的第三方 AppConn 系统发送前端或后台请求的服务能力。 + *
+ * 强烈推荐不要修改 AbstractOnlySSOAppConn 的体系结构,直接使用 DSS 系统默认提供的 OriginSSORequestServiceImpl + */ +public interface SSORequestService extends AppService { + + SSORequestOperation createSSORequestOperation(String appName); + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserCreationOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserCreationOperation.java new file mode 100644 index 000000000..fb089266a --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserCreationOperation.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user; + +import com.webank.wedatasphere.dss.standard.app.sso.user.ref.DSSUserContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.sso.user.ref.RefUserContentResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * @author enjoyyin + * @date 2022-04-25 + * @since 1.1.0 + */ +public abstract class SSOUserCreationOperation> + extends SSOUserOperation { + + /** + * 新增用户操作,当 DSS Admin 模块新建一个用户时,或是该用户第一次登陆到 DSS 时,同步请求第三方 AppConn 创建一个用户。 + *
+ * 如果创建成功,请在 ResponseRef 中返回对应的 refUserId。 + * DSS 不会存储该 refUserId,只所以希望第三方 AppConn 返回 refUserId,是因为 DSS 内嵌的应用工具可能会使用。 + * 如果您确认 DSS 内嵌的应用工具不会使用该 refUserId,您可以不用返回。 + * @param requestRef 请求创建用户 RequestRef + * @return 如果创建成功,请在 ResponseRef 中返回对应的 refUserId + * @throws ExternalOperationFailedException 创建失败时报错 + */ + public abstract RefUserContentResponseRef createUser(R requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserDeletionOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserDeletionOperation.java new file mode 100644 index 000000000..b4567e470 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserDeletionOperation.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user; + +import com.webank.wedatasphere.dss.standard.app.sso.user.ref.DSSUserContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * @author enjoyyin + * @date 2022-04-25 + * @since 1.1.0 + */ +public abstract class SSOUserDeletionOperation> + extends SSOUserOperation { + + /** + * 该方法暂时尚未被使用。 + * 删除用户操作,当 DSS Admin 模块删除一个用户时,同步请求第三方 AppConn 删除一个用户。 + * @param requestRef + * @return 如果删除成功,请返回 ResponseRef.newBuilder().success() + * @throws ExternalOperationFailedException 删除失败时抛出该异常 + */ + public abstract ResponseRef deleteUser(R requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserGetOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserGetOperation.java new file mode 100644 index 000000000..91443ab2d --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserGetOperation.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user; + +import com.webank.wedatasphere.dss.standard.app.sso.user.ref.DSSUserContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.sso.user.ref.RefUserContentResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * @author enjoyyin + * @date 2022-04-25 + * @since 1.1.0 + */ +public abstract class SSOUserGetOperation> + extends SSOUserOperation { + + /** + * 请求第三方 AppConn,获取唯一英文用户名为 username 的 第三方 AppConn 用户信息。 + * 如果第三方 AppConn 不存在该用户,请将 refUserId 置为 null。 + * @param requestRef + * @return 第三方 AppConn 用户信息,第三方 AppConn 不存在该用户,请将 refUserId 置为 null。 + * @throws ExternalOperationFailedException + */ + public abstract RefUserContentResponseRef getUser(R requestRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserOperation.java new file mode 100644 index 000000000..834685c86 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserOperation.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user; + +import com.webank.wedatasphere.dss.standard.app.sso.operation.AbstractOperation; +import com.webank.wedatasphere.dss.standard.app.sso.user.impl.SSOUserServiceImpl; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.service.Operation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author enjoyyin + * @date 2022-04-26 + * @since 1.1.0 + */ +public abstract class SSOUserOperation extends AbstractOperation + implements Operation { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + public final void setSSOUserService(SSOUserService service) { + this.service = service; + } + + @Override + protected String getAppConnName() { + if(service instanceof SSOUserServiceImpl) { + return ((SSOUserServiceImpl) service).getAppConnName(); + } + return null; + } +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserService.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserService.java new file mode 100644 index 000000000..11a9e1f6b --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserService.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user; + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppIntegrationService; + +/** + * DSS 用户与第三方 AppConn 的用户同步服务 + * @author enjoyyin + * @date 2022-04-25 + * @since 1.1.0 + */ +public interface SSOUserService extends AppIntegrationService { + + /** + * 用于请求第三方 AppConn 创建同名用户 + * @return + */ + SSOUserCreationOperation getSSOUserCreationOperation(); + + /** + * 用于修改第三方 AppConn 用户的基础信息 + * @return SSOUserUpdateOperation + */ + SSOUserUpdateOperation getSSOUserUpdateOperation(); + + /** + * 用于请求第三方 AppConn 获取同名用户信息 + * @return + */ + SSOUserGetOperation getSSOUserGetOperation(); + + /** + * 预留接口,用于删除第三方 AppConn 用户 + * @return + */ + SSOUserDeletionOperation getSSOUserDeletionOperation(); + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserUpdateOperation.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserUpdateOperation.java new file mode 100644 index 000000000..66230c8ae --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/SSOUserUpdateOperation.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user; + +import com.webank.wedatasphere.dss.standard.app.sso.user.ref.DSSUserContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * @author enjoyyin + * @date 2022-04-25 + * @since 1.1.0 + */ +public abstract class SSOUserUpdateOperation> + extends SSOUserOperation { + + /** + * 更新用户操作,当 DSS Admin 模块更新一个用户时,同步请求第三方 AppConn 更新一个用户。 + * @param requestRef + * @return 如果更新成功,请返回 ResponseRef.newBuilder().success() + * @throws ExternalOperationFailedException + */ + public abstract ResponseRef updateUser(R requestRef) throws ExternalOperationFailedException; + +} \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/impl/SSOUserServiceImpl.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/impl/SSOUserServiceImpl.java new file mode 100644 index 000000000..c37f9b342 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/impl/SSOUserServiceImpl.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user.impl; + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.app.sso.user.*; +import com.webank.wedatasphere.dss.standard.common.app.AppSingletonIntegrationServiceImpl; +import com.webank.wedatasphere.dss.standard.common.utils.AppStandardClassUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-04-26 + * @since 1.1.0 + */ +public class SSOUserServiceImpl extends AppSingletonIntegrationServiceImpl + implements SSOUserService { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + private String appConnName; + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + public String getAppConnName() { + return appConnName; + } + + protected T createSSOUserOperation(Class clazz) { + List> classes = AppStandardClassUtils.getInstance(appConnName).getClasses(clazz); + if(CollectionUtils.isEmpty(classes)) { + logger.info("no implementation find for {}, ignore to sync user to AppConn {}.", clazz.getSimpleName(), appConnName); + return null; + } + T instance = AppStandardClassUtils.getInstance(appConnName).getInstanceOrWarn(clazz); + logger.info("find a implementation {} for {}, ignore to sync user to AppConn {}.", instance.getClass().getName(), + clazz.getSimpleName(), appConnName); + return instance; + } + + @Override + public SSOUserCreationOperation getSSOUserCreationOperation() { + return getOrCreate(() -> createSSOUserOperation(SSOUserCreationOperation.class), SSOUserCreationOperation.class); + } + + @Override + public SSOUserUpdateOperation getSSOUserUpdateOperation() { + return getOrCreate(() -> createSSOUserOperation(SSOUserUpdateOperation.class), SSOUserUpdateOperation.class); + } + + @Override + public SSOUserGetOperation getSSOUserGetOperation() { + return getOrCreate(() -> createSSOUserOperation(SSOUserGetOperation.class), SSOUserGetOperation.class); + } + + @Override + public SSOUserDeletionOperation getSSOUserDeletionOperation() { + return getOrCreate(() -> createSSOUserOperation(SSOUserDeletionOperation.class), SSOUserDeletionOperation.class); + } + + @Override + protected void initOperation(SSOUserOperation operation) { + super.initOperation(operation); + operation.setSSOUserService(this); + } +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/ref/DSSUserContentRequestRef.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/ref/DSSUserContentRequestRef.java new file mode 100644 index 000000000..6904a2b3f --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/ref/DSSUserContentRequestRef.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user.ref; + +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.ref.WorkspaceRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRefImpl; + +/** + * @author enjoyyin + * @date 2022-04-25 + * @since 1.1.0 + */ +public interface DSSUserContentRequestRef> + extends WorkspaceRequestRef { + + default User getUser() { + return (User) getParameter("user"); + } + + default R setUser(User user) { + setParameter("user", user); + return (R) this; + } + + /** + * 请注意,该方法返回的 workspace 不会带有 workspaceId 和 workspaceName。 + * 因为用户的 CRUD 与工作空间无关的,该 workspace 只是为了保证可以正常使用 SSORequestOperation + * @return + */ + @Override + default Workspace getWorkspace() { + return (Workspace) getParameter("workspace"); + } + + default R setWorkspace(Workspace workspace) { + setParameter("workspace", workspace); + return (R) this; + } + + interface User { + + /** + * 英文用户名,全局唯一 + * @return 英文用户名 + */ + String getUsername(); + + /** + * 中文用户名,允许重复 + * @return 中文用户名 + */ + String getName(); + + /** + * 是否是第一次登陆 DSS,一般为 true + * @return 如果是第一次登陆 DSS,则为true + */ + Boolean isFirstLogin(); + + /** + * 是否为超级管理员用户 + * @return 如果是超级管理员,则返回true + */ + Boolean isAdmin(); + + } + + class DSSUserContentRequestRefImpl extends RequestRefImpl + implements DSSUserContentRequestRef { + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/ref/RefUserContentResponseRef.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/ref/RefUserContentResponseRef.java new file mode 100644 index 000000000..9cfd218d7 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/sso/user/ref/RefUserContentResponseRef.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.dss.standard.app.sso.user.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-04-25 + * @since 1.1.0 + */ +public interface RefUserContentResponseRef extends ResponseRef { + + /** + * 第三方 AppConn 的 userId + * @return + */ + String getRefUserId(); + + /** + * 第三方 AppConn 的额外用户信息,可以为空 + * @return + */ + Map getRefUserContent(); + + static Builder newBuilder() { + return new Builder(); + } + + class Builder extends ResponseRefBuilder.ExternalResponseRefBuilder { + + private String refUserId; + private Map refUserContent; + + public Builder setRefUserId(String refUserId) { + this.refUserId = refUserId; + return this; + } + + public Builder setRefUserContent(Map refUserContent) { + this.refUserContent = refUserContent; + return this; + } + + private class RefUserContentResponseRefImpl extends ResponseRefImpl implements RefUserContentResponseRef { + + public RefUserContentResponseRefImpl() { + super(Builder.this.responseBody, Builder.this.status, Builder.this.errorMsg, Builder.this.responseMap); + } + + @Override + public String getRefUserId() { + return refUserId; + } + + @Override + public Map getRefUserContent() { + return refUserContent; + } + } + + @Override + protected RefUserContentResponseRef createResponseRef() { + return new RefUserContentResponseRefImpl(); + } + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/sso/utils/SSOHelper.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/sso/utils/SSOHelper.java new file mode 100644 index 000000000..1f965486d --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/sso/utils/SSOHelper.java @@ -0,0 +1,193 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.sso.utils; + +import com.webank.wedatasphere.dss.common.conf.DSSCommonConf; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOBuilderService; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardWarnException; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Arrays; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class SSOHelper { + + private static final String WORKSPACE_ID_COOKIE_KEY = "workspaceId"; + private static final String WORKSPACE_NAME_COOKIE_KEY = "workspaceName"; + private static SSOBuilderService ssoBuilderService; + + private static final Logger LOGGER = LoggerFactory.getLogger(SSOHelper.class); + + public static void setSSOBuilderService(SSOBuilderService ssoBuilderService) { + if(ssoBuilderService == null) { + return; + } + SSOHelper.ssoBuilderService = ssoBuilderService; + } + + public static Workspace getWorkspace(Map cookies) { + Workspace workspace = new Workspace(); + workspace.setCookies(cookies); + if(cookies.isEmpty() || !cookies.containsKey(WORKSPACE_NAME_COOKIE_KEY)) { + throw new AppStandardWarnException(50010, "Cannot find workspace info from cookies map, please ensure cookies have transferred correctly."); + } + String workspaceName = cookies.get(WORKSPACE_NAME_COOKIE_KEY); + workspace.setWorkspaceName(workspaceName); + if(cookies.containsKey(WORKSPACE_ID_COOKIE_KEY)) { + long workspaceId = Long.parseLong(cookies.get(WORKSPACE_ID_COOKIE_KEY)); + workspace.setWorkspaceId(workspaceId); + } + return workspace; + } + + public static void addWorkspaceInfo(HttpServletRequest request, Workspace workspace) { + String gatewayUrl = request.getHeader("GATEWAY_URL"); + if(StringUtils.isNotBlank(gatewayUrl)) { + if(gatewayUrl.startsWith("http")) { + workspace.setDssUrl(gatewayUrl); + } else { + workspace.setDssUrl("http://" + gatewayUrl); + } + } + Cookie[] cookies = request.getCookies(); + Arrays.stream(cookies).forEach(cookie -> workspace.addCookie(cookie.getName(), cookie.getValue())); + if(StringUtils.isBlank(workspace.getWorkspaceName())) { + throw new AppStandardWarnException(50010, "Cannot find workspace info from cookies, please ensure front-web has injected cookie['workspaceName'](不能找到工作空间名,请确认前端是否已经注入cookie['workspaceName'])."); + } + if(workspace.getWorkspaceId() <= 0) { + throw new AppStandardWarnException(50010, "Cannot find workspace info from cookies, please ensure front-web has injected cookie['workspaceId'](不能找到工作空间名,请确认前端是否已经注入cookie['workspaceId'])."); + } + } + + public static Workspace getWorkspace(HttpServletRequest request){ + Workspace workspace = new Workspace(); + Cookie[] cookies = request.getCookies(); + Arrays.stream(cookies).forEach(cookie -> { + if(WORKSPACE_NAME_COOKIE_KEY.equals(cookie.getName())) { + workspace.setWorkspaceName(cookie.getValue()); + } else if(WORKSPACE_ID_COOKIE_KEY.equals(cookie.getName())) { + workspace.setWorkspaceId(Long.parseLong(cookie.getValue())); + } + }); + addWorkspaceInfo(request, workspace); + return workspace; + } + + public static Workspace setAndGetWorkspace(HttpServletRequest request, HttpServletResponse response, long workspaceId, String workspaceName) { + boolean isWorkspaceExists = Arrays.stream(request.getCookies()) + .anyMatch(cookie -> WORKSPACE_ID_COOKIE_KEY.equals(cookie.getName()) && workspaceId == Long.parseLong(cookie.getValue())) && + Arrays.stream(request.getCookies()) + .anyMatch(cookie -> WORKSPACE_NAME_COOKIE_KEY.equals(cookie.getName()) && workspaceName.equals(cookie.getValue())); + if (isWorkspaceExists) { + LOGGER.warn("workspace {} already exists in DSS cookies, ignore to set it again.", workspaceName); + return getWorkspace(request); + } + String workspaceIdStr = String.valueOf(workspaceId); + String domain = getCookieDomain(request.getHeader("Referer")); + Cookie workspaceIdCookie = new Cookie(WORKSPACE_ID_COOKIE_KEY, workspaceIdStr); + workspaceIdCookie.setPath("/"); +// workspaceIdCookie.setDomain(domain); + Cookie workspaceNameCookie = new Cookie(WORKSPACE_NAME_COOKIE_KEY, workspaceName); + workspaceNameCookie.setPath("/"); +// workspaceNameCookie.setDomain(domain); + response.addCookie(workspaceIdCookie); + response.addCookie(workspaceNameCookie); + Workspace workspace = new Workspace(); + workspace.setWorkspaceId(workspaceId); + workspace.setWorkspaceName(workspaceName); + addWorkspaceInfo(request, workspace); + LOGGER.info("Try to change the workspace to [{}] in DSS cookies of domain({}), the workspace info is {}.", + workspaceName, domain, workspace); + workspace.setWorkspaceId(workspaceId); + workspace.setWorkspaceName(workspaceName); + workspace.addCookie(WORKSPACE_ID_COOKIE_KEY, workspaceIdStr); + workspace.addCookie(WORKSPACE_NAME_COOKIE_KEY, workspaceName); + return workspace; + } + + public static SSOUrlBuilderOperation createSSOUrlBuilderOperation(Workspace workspace) { + SSOUrlBuilderOperation operation = ssoBuilderService.createSSOUrlBuilderOperation(); + setSSOUrlBuilderOperation(operation, workspace); + return operation; + } + + public static void setSSOUrlBuilderOperation(SSOUrlBuilderOperation operation, Workspace workspace) { + workspace.getCookies().forEach(operation::addCookie); + operation.setDSSUrl(workspace.getDssUrl()); + operation.setWorkspace(workspace.getWorkspaceName()); + } + + private static final Pattern DOMAIN_REGEX = Pattern.compile("[a-zA-Z][a-zA-Z0-9\\.]+"); + private static final Pattern IP_REGEX = Pattern.compile("([^:]+):.+"); + + /** + * "dss.com" -> ".dss.com" + * "127.0.0.1" -> "127.0.0.1" + * "127.0.0.1:8080" -> "127.0.0.1" + * @param host the Host in HttpRequest Headers + * @return + */ + public static String getCookieDomain(String host) { + int level = DSSCommonConf.DSS_DOMAIN_LEVEL.getValue(); + if(host.startsWith("https://")) { + host = host.substring(8); + } else if(host.startsWith("http://")) { + host = host.substring(7); + } + if (host.endsWith("/")) { + host = host.substring(0, host.length() - 1); + } + if(DOMAIN_REGEX.matcher(host).find()) { + String[] domains = host.split("\\."); + int index = level; + if (domains.length == level) { + index = level - 1; + } else if (domains.length < level) { + index = domains.length; + } + if (index < 0) { + return host; + } + String[] parsedDomains = Arrays.copyOfRange(domains, index, domains.length); + if (parsedDomains.length < level) { + return host; + } + String domain = String.join(".", parsedDomains); + if(domains.length >= level) { + return "." + domain; + } + return domain; + } + Matcher matcher = IP_REGEX.matcher(host); + if(matcher.find()) { + return matcher.group(1); + } else { + return host; + } + } + +} diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/plugin/DssMsgCacheOperationImpl.scala b/dss-standard/sso-standard/sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/plugin/DssMsgCacheOperationImpl.scala new file mode 100644 index 000000000..f6d941fb2 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/plugin/DssMsgCacheOperationImpl.scala @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin + +import java.lang +import java.lang.reflect.Type + +import com.google.gson.{GsonBuilder, JsonElement, JsonPrimitive, JsonSerializationContext, JsonSerializer} +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg +import com.webank.wedatasphere.dss.standard.app.sso.builder.impl.DSSMsgImpl +import javax.servlet.http.HttpServletRequest + + +class DssMsgCacheOperationImpl private() extends DssMsgCacheOperation { + + override def setWorkspaceToSession(req: HttpServletRequest, workspaceName: String): Unit = + getDSSMsgInSession(req) match { + case dssMsg: DSSMsgImpl => dssMsg.setWorkspaceName(workspaceName) + case _ => + } + + override def getWorkspaceInSession(req: HttpServletRequest): String = + getDSSMsgInSession(req) match { + case dssMsg: DSSMsgImpl => dssMsg.getWorkspaceName + case _ => null + } + + override def getDSSMsgInSession(request: HttpServletRequest): DssMsgBuilderOperation.DSSMsg = + request.getSession.getAttribute(DssMsgCacheOperationImpl.DSS_MSG_KEY) match { + case dssMsg: DSSMsgImpl => dssMsg + case _ => null + } + + override def setDSSMsgToSession(dssMsg: DssMsgBuilderOperation.DSSMsg, request: HttpServletRequest): Unit = { + val newDSSMsg = dssMsg match { + case dss: DSSMsgImpl => + dss.setRedirectUrl(null) + dss + case dss: DSSMsg => + val dssMsg1 = new DSSMsgImpl + dssMsg1.setCookies(dss.getCookies) + dssMsg1.setDSSUrl(dss.getDSSUrl) + dssMsg1.setWorkspaceName(dss.getWorkspaceName) + dssMsg1 + } + request.getSession.setAttribute(DssMsgCacheOperationImpl.DSS_MSG_KEY, newDSSMsg) + } +} +object DssMsgCacheOperationImpl { + private val DSS_MSG_KEY = "dss_msg_key" + + private val dssMsgCacheOperation = new DssMsgCacheOperationImpl + + def getDssMsgCacheOperation: DssMsgCacheOperation = dssMsgCacheOperation +} \ No newline at end of file diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOIntegrationConf.scala b/dss-standard/sso-standard/sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOIntegrationConf.scala new file mode 100644 index 000000000..8f90830e6 --- /dev/null +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/scala/com/webank/wedatasphere/dss/standard/app/sso/plugin/SSOIntegrationConf.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.sso.plugin + +import java.lang +import java.lang.reflect.Type + +import com.google.gson.{GsonBuilder, JsonElement, JsonPrimitive, JsonSerializationContext, JsonSerializer} + + +object SSOIntegrationConf { + implicit val gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls + .registerTypeAdapter(classOf[java.lang.Double], new JsonSerializer[java.lang.Double] { + override def serialize(t: lang.Double, `type`: Type, jsonSerializationContext: JsonSerializationContext): JsonElement = + if(t == t.longValue()) new JsonPrimitive(t.longValue()) else new JsonPrimitive(t) + }).create +} diff --git a/dss-standard/structure-standard/dss-project-plugin/pom.xml b/dss-standard/structure-standard/dss-project-plugin/pom.xml new file mode 100644 index 000000000..8923fa4f5 --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/pom.xml @@ -0,0 +1,67 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-project-plugin + + + + com.webank.wedatasphere.dss + dss-origin-sso-integration-standard + ${dss.version} + compile + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + + + \ No newline at end of file diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectAuth.java b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectAuth.java new file mode 100644 index 000000000..fcb0f41ac --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectAuth.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin; + +import java.util.List; + + +public interface ProjectAuth { + + String getWorkspaceName(); + + String getProjectId(); + + String getProjectName(); + + List getEditUsers(); + + List getAccessUsers(); + + List getExecuteUsers(); + + default List getDeleteUsers() { + return getEditUsers(); + } + +} diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectCooperationPlugin.java b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectCooperationPlugin.java new file mode 100644 index 000000000..aff15ef1e --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectCooperationPlugin.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin; + +import javax.servlet.http.HttpServletRequest; + + +public interface ProjectCooperationPlugin { + + ProjectAuth getProjectAuth(HttpServletRequest request, String projectId); + + ProjectAuth getProjectAuthByName(HttpServletRequest request, String projectName); + +} diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectPlugin.java b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectPlugin.java new file mode 100644 index 000000000..4f7e64900 --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/ProjectPlugin.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin; + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.origin.OriginSSOIntegrationStandardFactory; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + + +public interface ProjectPlugin { + + //TODO 如果一个用户从dss空间进入到qualitis建立工程,添加一个是否是创建工程的请求,添加一个开关参数,是否允许用户建工程 + List getProjects(HttpServletRequest request); + + default List filterProjects(List projects, HttpServletRequest request, + Function getProjectId) { + SSOIntegrationStandard ssoIntegrationStandard = new OriginSSOIntegrationStandardFactory().getSSOIntegrationStandard(); + if(!ssoIntegrationStandard.getSSOPluginService().createSSOMsgParseOperation().isDssRequest(request)) { + return projects; + } + List projectList = getProjects(request); + if(projectList == null) { + return new ArrayList<>(); + } + List filteredList = projects.stream().filter(t -> projectList.contains(getProjectId.apply(t))) + .collect(Collectors.toList()); + return filteredList; + } + +} diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/AbstractProjectAuthInterceptor.java b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/AbstractProjectAuthInterceptor.java new file mode 100644 index 000000000..10ca96d68 --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/AbstractProjectAuthInterceptor.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.filter; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.ProjectAuth; +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardWarnException; + +import java.util.ArrayList; +import java.util.List; +import javax.servlet.http.HttpServletRequest; + + +public abstract class AbstractProjectAuthInterceptor implements ProjectAuthInterceptor { + + private List projectUris = new ArrayList<>(); + + @Override + public boolean isProjectRequest(HttpServletRequest request) { + String requestUri = request.getRequestURI(); + return projectUris.stream().anyMatch(requestUri::matches); + } + + protected void addProjectUri(String projectUri) { + projectUris.add(projectUri); + } + + @Override + public String getForbiddenMsg(ProjectAuth projectAuth, ProjectRequestType projectRequestType, + HttpServletRequest request) { + String message = "You have no permission to " + projectRequestType.toString().toLowerCase() + + " the content of project " + projectAuth.getProjectName(); + Object returnObj = getForbiddenMsg(message); + if(returnObj instanceof String) { + return (String) returnObj; + } else { + try { + return HttpClient.objectMapper().writeValueAsString(returnObj); + } catch (JsonProcessingException e) { + throw new AppStandardWarnException(50002, "Serialize object to string failed! Object: " + returnObj, e); + } + } + } + + protected abstract Object getForbiddenMsg(String message); +} diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectAuthInterceptor.java b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectAuthInterceptor.java new file mode 100644 index 000000000..d17469ef2 --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectAuthInterceptor.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.filter; + +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.ProjectAuth; + +import javax.servlet.http.HttpServletRequest; + + +public interface ProjectAuthInterceptor { + + boolean isProjectRequest(HttpServletRequest request); + + String getProjectId(HttpServletRequest request); + + String getProjectName(HttpServletRequest request); + + ProjectRequestType getProjectRequestType(HttpServletRequest request); + + String getForbiddenMsg(ProjectAuth projectAuth, ProjectRequestType projectRequestType, + HttpServletRequest request); + +} diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectRequestType.java b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectRequestType.java new file mode 100644 index 000000000..3c34346d4 --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectRequestType.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.filter; + + +public enum ProjectRequestType { + Edit, Access, Delete, Execute +} diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthImpl.java b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthImpl.java new file mode 100644 index 000000000..062deea85 --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin; + +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.ProjectAuth; + +import java.util.List; + + +public class ProjectAuthImpl implements ProjectAuth { + + private String workspaceName; + private String projectId; + private String projectName; + private List editUsers; + private List accessUsers; + private List deleteUsers; + private List executeUsers; + + @Override + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + @Override + public String getProjectId() { + return projectId; + } + + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + @Override + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + @Override + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + @Override + public List getAccessUsers() { + return accessUsers; + } + + @Override + public List getExecuteUsers() { + return executeUsers; + } + + public void setExecuteUsers(List executeUsers) { + this.executeUsers = executeUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + @Override + public List getDeleteUsers() { + if(deleteUsers == null || deleteUsers.isEmpty()) { + return editUsers; + } + return deleteUsers; + } + + public void setDeleteUsers(List deleteUsers) { + this.deleteUsers = deleteUsers; + } + + @Override + public String toString() { + return "ProjectAuthImpl(" + + "workspaceName='" + workspaceName + '\'' + + ", projectId='" + projectId + '\'' + + ", projectName='" + projectName + '\'' + + ", editUsers=" + editUsers + + ", accessUsers=" + accessUsers + + ", deleteUsers=" + deleteUsers + + ')'; + } +} diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/conf/ProjectCooperateConfiguration.scala b/dss-standard/structure-standard/dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/conf/ProjectCooperateConfiguration.scala new file mode 100644 index 000000000..ff4691953 --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/conf/ProjectCooperateConfiguration.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.conf + +import org.apache.linkis.common.conf.{CommonVars, TimeType} + + +object ProjectCooperateConfiguration { + + val EXPIRE_PROJECT_AUTH_TIMEOUT = CommonVars("wds.dss.project.auth.timeout", new TimeType("10m")) + val EXPIRE_PROJECT_AUTH_SCAN_INTERVAL = CommonVars("wds.dss.project.auth.scan.interval", new TimeType("5m")) + +} diff --git a/dss-standard/structure-standard/dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectCooperationFilter.scala b/dss-standard/structure-standard/dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectCooperationFilter.scala new file mode 100644 index 000000000..7882b6d83 --- /dev/null +++ b/dss-standard/structure-standard/dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectCooperationFilter.scala @@ -0,0 +1,167 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.filter + +import java.util +import java.util.concurrent.TimeUnit + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard +import com.webank.wedatasphere.dss.standard.app.sso.origin.OriginSSOIntegrationStandardFactory +import com.webank.wedatasphere.dss.standard.app.sso.plugin.filter.UserInterceptor +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.conf.ProjectCooperateConfiguration +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.filter.ProjectRequestType.{Access, Delete, Edit, Execute} +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.{ProjectAuth, ProjectCooperationPlugin} +import org.apache.linkis.common.conf.Configuration +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.httpclient.exception.HttpClientResultException +import javax.servlet._ +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} +import org.apache.commons.lang.StringUtils + + +abstract class ProjectCooperationFilter extends Filter with Logging { + + protected var projectAuthInterceptor: ProjectAuthInterceptor = _ + protected var userInterceptor: UserInterceptor = _ + protected var projectCooperationPlugin: ProjectCooperationPlugin = _ + private val cachedProjectAuthsWithId = new util.HashMap[String, TimeoutProjectAuth]() + private val cachedProjectAuthsWithName = new util.HashMap[String, TimeoutProjectAuth]() + private val ignoreProjectWithId = new util.ArrayList[String]() + private val ignoreProjectWithName = new util.ArrayList[String]() + + override def init(filterConfig: FilterConfig): Unit = { + projectAuthInterceptor = getProjectAuthInterceptor(filterConfig) + userInterceptor = getUserAuthInterceptor(filterConfig) + projectCooperationPlugin = getProjectCooperationPlugin(filterConfig) + } + + protected def getProjectAuthInterceptor(filterConfig: FilterConfig): ProjectAuthInterceptor + protected def getUserAuthInterceptor(filterConfig: FilterConfig): UserInterceptor + protected def getProjectCooperationPlugin(filterConfig: FilterConfig): ProjectCooperationPlugin + + override def doFilter(servletRequest: ServletRequest, servletResponse: ServletResponse, + filterChain: FilterChain): Unit = { + val req = servletRequest.asInstanceOf[HttpServletRequest] + val uri = req.getRequestURI + val ssoIntegrationStandard:SSOIntegrationStandard =new OriginSSOIntegrationStandardFactory().getSSOIntegrationStandard + if(ssoIntegrationStandard.getSSOPluginService + .createDssMsgCacheOperation().getWorkspaceInSession(req) == null) { + info(s"Request $uri has no relation with DSS, ignore it.") + filterChain.doFilter(servletRequest, servletResponse) + return + } + val resp = servletResponse.asInstanceOf[HttpServletResponse] + if(!projectAuthInterceptor.isProjectRequest(req)) { + info(s"Request $uri is not a project request, ignore it.") + filterChain.doFilter(req, resp) + return + } + val projectId = projectAuthInterceptor.getProjectId(req) + if(ignoreProjectWithId.contains(projectId)) { + info(s"The project $projectId is not a DSS project, ignore it, request is $uri.") + filterChain.doFilter(req, resp) + return + } + val projectAuth = if(StringUtils.isNotBlank(projectId)) { + getProjectAuthById(req, projectId) + } else { + val projectName = projectAuthInterceptor.getProjectName(req) + // If this request cannot get the projectId or projectName, ignore it. + if(StringUtils.isBlank(projectName) || ignoreProjectWithName.contains(projectName)) { + info(s"This request $uri cannot get the projectId or projectName, ignore it.") + filterChain.doFilter(req, resp) + return + } + getProjectAuthByName(req, projectName) + } + if(projectAuth == null) { + filterChain.doFilter(req, resp) + return + } + val projectRequestType = projectAuthInterceptor.getProjectRequestType(req) + debug(s"RequestURI: $uri, ProjectRequestType: $projectRequestType, ProjectAuth: $projectAuth.") + val user = userInterceptor.getUser(req) + val passed = projectRequestType match { + case Edit => projectAuth.getEditUsers.contains(user) + case Access => projectAuth.getAccessUsers.contains(user) + case Delete => projectAuth.getDeleteUsers.contains(user) + case Execute => projectAuth.getDeleteUsers.contains(user) + } + if(passed) { + info(s"Cooperation Project request $uri has passed auth validation.") + filterChain.doFilter(req, resp) + } else { + val msg = projectAuthInterceptor.getForbiddenMsg(projectAuth, projectRequestType, req) + info(s"Cooperation Project request $uri has been failed with auth validation.") + resp.getOutputStream.write(msg.getBytes(Configuration.BDP_ENCODING.getValue)) + resp.setStatus(403) + resp.getOutputStream.flush() + } + } + + protected def getProjectAuthById(req: HttpServletRequest, projectId: String): ProjectAuth = + getProjectAuth(projectId, cachedProjectAuthsWithId, + projectCooperationPlugin.getProjectAuth(req, projectId), ignoreProjectWithId) + + protected def getProjectAuthByName(req: HttpServletRequest, projectName: String): ProjectAuth = + getProjectAuth(projectName, cachedProjectAuthsWithName, + projectCooperationPlugin.getProjectAuthByName(req, projectName), ignoreProjectWithName) + + private def getProjectAuth(key: String, + map: util.HashMap[String, TimeoutProjectAuth], + getNewProjectAuth: => ProjectAuth, + list: util.ArrayList[String]): ProjectAuth = { + var timeoutProjectAuth = map.get(key) + if(timeoutProjectAuth == null || timeoutProjectAuth.isTimeout) { + key.intern().synchronized { + timeoutProjectAuth = map.get(key) + if(timeoutProjectAuth == null || timeoutProjectAuth.isTimeout) { + val projectAuth = Utils.tryCatch(getNewProjectAuth) { + case t: HttpClientResultException => + if(t.getDesc.contains("not exists")) { + list.add(key) + return null + } else throw t + case t: Throwable => throw t + } + map.put(key, TimeoutProjectAuth(projectAuth)) + } + } + } + map.get(key).projectAuth + } + + override def destroy(): Unit = {} + + private case class TimeoutProjectAuth(createTime: Long, projectAuth: ProjectAuth) { + def isTimeout: Boolean = System.currentTimeMillis() - createTime > ProjectCooperateConfiguration.EXPIRE_PROJECT_AUTH_TIMEOUT.getValue.toLong + } + private object TimeoutProjectAuth { + def apply(projectAuth: ProjectAuth): TimeoutProjectAuth = + new TimeoutProjectAuth(System.currentTimeMillis(), projectAuth) + } + + Utils.defaultScheduler.scheduleAtFixedRate(new Runnable { + override def run(): Unit = { + cachedProjectAuthsWithId.keySet().toArray + .foreach{ case k: String => if(cachedProjectAuthsWithId.get(k).isTimeout) cachedProjectAuthsWithId.remove(k)} + cachedProjectAuthsWithName.keySet().toArray + .foreach{ case k: String => if(cachedProjectAuthsWithName.get(k).isTimeout) cachedProjectAuthsWithName.remove(k)} + } + } , ProjectCooperateConfiguration.EXPIRE_PROJECT_AUTH_SCAN_INTERVAL.getValue.toLong, ProjectCooperateConfiguration.EXPIRE_PROJECT_AUTH_SCAN_INTERVAL.getValue.toLong, TimeUnit.MILLISECONDS) + +} diff --git a/dss-standard/structure-standard/dss-role-plugin/pom.xml b/dss-standard/structure-standard/dss-role-plugin/pom.xml new file mode 100644 index 000000000..841f66742 --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/pom.xml @@ -0,0 +1,66 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-role-plugin + + + + com.webank.wedatasphere.dss + dss-origin-sso-integration-standard + ${dss.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + + + \ No newline at end of file diff --git a/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/DSSUserRoles.java b/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/DSSUserRoles.java new file mode 100644 index 000000000..3294a1b76 --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/DSSUserRoles.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.plugin.role; + +import java.util.List; + + +public interface DSSUserRoles { + + List getRoles(); + + String getWorkspaceName(); + + String getUser(); + +} diff --git a/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/RolePlugin.java b/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/RolePlugin.java new file mode 100644 index 000000000..47ab3d26d --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/RolePlugin.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.plugin.role; + +import com.webank.wedatasphere.dss.standard.app.structure.plugin.role.impl.OriginRolePlugin; + +import java.util.List; +import javax.servlet.http.HttpServletRequest; + + +public interface RolePlugin { + + DSSUserRoles getRoles(HttpServletRequest request); + + List getUsers(HttpServletRequest request, String role); + + static RolePlugin getRolePlugin() { + return OriginRolePlugin.getRolePlugin(); + } + +} diff --git a/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/DSSUserRolesImpl.java b/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/DSSUserRolesImpl.java new file mode 100644 index 000000000..2d5e256d9 --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/DSSUserRolesImpl.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.plugin.role.impl; + +import com.webank.wedatasphere.dss.standard.app.structure.plugin.role.DSSUserRoles; + +import java.util.List; + + +public class DSSUserRolesImpl implements DSSUserRoles { + + private List roles; + private String workspaceName; + private String user; + + public void setRoles(List roles) { + this.roles = roles; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public void setUser(String user) { + this.user = user; + } + + @Override + public List getRoles() { + return roles; + } + + @Override + public String getWorkspaceName() { + return workspaceName; + } + + @Override + public String getUser() { + return user; + } +} diff --git a/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/OriginRolePlugin.scala b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/OriginRolePlugin.scala new file mode 100644 index 000000000..307b7ac5c --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/OriginRolePlugin.scala @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.plugin.role.impl + +import java.util + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard +import com.webank.wedatasphere.dss.standard.app.sso.origin.OriginSSOIntegrationStandardFactory +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient +import com.webank.wedatasphere.dss.standard.app.structure.plugin.role.{DSSUserRoles, RolePlugin} +import javax.servlet.http.HttpServletRequest + + +class OriginRolePlugin private() extends RolePlugin { + val ssoIntegrationStandard:SSOIntegrationStandard =new OriginSSOIntegrationStandardFactory().getSSOIntegrationStandard + + override def getRoles(request: HttpServletRequest): DSSUserRoles = { + val dssMsg = ssoIntegrationStandard.getSSOPluginService + .createDssMsgCacheOperation().getDSSMsgInSession(request) + val dwsHttpClient = HttpClient.getDSSClient(dssMsg.getDSSUrl) + val dssPrivilege = new DSSUserRolesImpl + dssPrivilege.setWorkspaceName(dssMsg.getWorkspaceName) + val userRoleInfoAction = new RoleInfoOfUserAction + userRoleInfoAction.setWorkspace(dssMsg.getWorkspaceName) + HttpClient.addCookies(dssMsg, userRoleInfoAction) + dwsHttpClient.execute(userRoleInfoAction) match { + case userRoleInfoResult: RoleInfoOfUserResult => + dssPrivilege.setRoles(userRoleInfoResult.getRoles) + dssPrivilege.setUser(userRoleInfoResult.getUser) + } + dssPrivilege + } + + override def getUsers(request: HttpServletRequest, role: String): util.List[String] = { + val dssMsg = ssoIntegrationStandard.getSSOPluginService + .createDssMsgCacheOperation().getDSSMsgInSession(request) + val dwsHttpClient = HttpClient.getDSSClient(dssMsg.getDSSUrl) + val userInfoOfRole = new UserInfoOfRoleAction + userInfoOfRole.setWorkspace(dssMsg.getWorkspaceName) + HttpClient.addCookies(dssMsg, userInfoOfRole) + dwsHttpClient.execute(userInfoOfRole) match { + case result: UserInfoOfRoleResult => + result.getUsers + } + } +} +object OriginRolePlugin { + private val rolePlugin = new OriginRolePlugin + def getRolePlugin: RolePlugin = rolePlugin +} \ No newline at end of file diff --git a/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/RoleInfoOfUserAction.scala b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/RoleInfoOfUserAction.scala new file mode 100644 index 000000000..806af7ec7 --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/RoleInfoOfUserAction.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.plugin.role.impl + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.POSTAction + + +class RoleInfoOfUserAction extends POSTAction with DWSHttpAction { + + override def suffixURLs: Array[String] = Array("dss", "getRolesOfWorkspaceUser") + + override def getRequestPayload: String = "" + + def setWorkspace(workspace: String): Unit = addRequestPayload("workspaceName", workspace) + +} diff --git a/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/RoleInfoOfUserResult.scala b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/RoleInfoOfUserResult.scala new file mode 100644 index 000000000..072b8a393 --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/RoleInfoOfUserResult.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.plugin.role.impl + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/getRolesOfWorkspaceUser") +class RoleInfoOfUserResult extends DWSResult { + + @BeanProperty var user: String = _ + + @BeanProperty var roles: java.util.List[String] = _ + +} diff --git a/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/UserInfoOfRoleAction.scala b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/UserInfoOfRoleAction.scala new file mode 100644 index 000000000..3aed27103 --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/UserInfoOfRoleAction.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.plugin.role.impl + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.POSTAction + + +class UserInfoOfRoleAction extends POSTAction with DWSHttpAction { + + override def suffixURLs: Array[String] = Array("dss", "getUsersOfWorkspaceRole") + + def setWorkspace(workspace: String): Unit = addRequestPayload("workspaceName", workspace) + + override def getRequestPayload: String = "" +} \ No newline at end of file diff --git a/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/UserInfoOfRoleResult.scala b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/UserInfoOfRoleResult.scala new file mode 100644 index 000000000..51a197a50 --- /dev/null +++ b/dss-standard/structure-standard/dss-role-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/plugin/role/impl/UserInfoOfRoleResult.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.plugin.role.impl + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/getUsersOfWorkspaceRole") +class UserInfoOfRoleResult extends DWSResult { + + @BeanProperty var users: java.util.List[String] = _ + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml b/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml new file mode 100644 index 000000000..47c8e97b0 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml @@ -0,0 +1,43 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + dss-structure-integration-standard + + + org.apache.linkis + linkis-common + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-sso-integration-standard + ${dss.version} + + + + \ No newline at end of file diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureIntegrationStandard.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureIntegrationStandard.java new file mode 100644 index 000000000..d691e87d5 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureIntegrationStandard.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure; + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.dss.standard.app.structure.role.RoleService; +import com.webank.wedatasphere.dss.standard.app.structure.status.AppStatusService; +import com.webank.wedatasphere.dss.standard.common.core.AbstractAppIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +public abstract class AbstractStructureIntegrationStandard extends AbstractAppIntegrationStandard + implements StructureIntegrationStandard { + + protected abstract ProjectService createProjectService(); + + protected RoleService createRoleService() { + return null; + } + + protected AppStatusService createAppStatusService() { + return null; + } + + @Override + protected void initService(T service) { + service.setAppStandard(this); + } + + @Override + public void init() { + } + + @Override + public final RoleService getRoleService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createRoleService, RoleService.class); + } + + @Override + public final ProjectService getProjectService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createProjectService, ProjectService.class); + } + + @Override + public final AppStatusService getAppStateService(AppInstance appInstance) { + return getOrCreate(appInstance, this::createAppStatusService, AppStatusService.class); + } + + @Override + public void close() { + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureOperation.java new file mode 100644 index 000000000..ff7b12d07 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureOperation.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.dss.standard.app.structure; + +import com.webank.wedatasphere.dss.standard.app.sso.operation.AbstractOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public abstract class AbstractStructureOperation + extends AbstractOperation implements StructureOperation { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + public final void setStructureService(StructureService service) { + this.service = service; + } + + protected StructureService getStructureService() { + return (StructureService) this.service; + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureService.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureService.java new file mode 100644 index 000000000..382f5a03c --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/AbstractStructureService.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure; + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppSingletonIntegrationServiceImpl; + +public class AbstractStructureService extends AppSingletonIntegrationServiceImpl implements StructureService { + + private StructureIntegrationStandard appStandard; + + @Override + protected void initOperation(StructureOperation operation) { + operation.setStructureService(this); + super.initOperation(operation); + } + + @Override + public void setAppStandard(StructureIntegrationStandard appStandard) { + this.appStandard = appStandard; + } + + @Override + public StructureIntegrationStandard getAppStandard() { + return appStandard; + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/OptionalIntegrationStandard.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/OptionalIntegrationStandard.java new file mode 100644 index 000000000..aa2b598a5 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/OptionalIntegrationStandard.java @@ -0,0 +1,74 @@ +package com.webank.wedatasphere.dss.standard.app.structure; + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.app.structure.optional.OptionalService; +import com.webank.wedatasphere.dss.standard.common.core.AbstractAppIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; + +import java.util.HashMap; +import java.util.Map; + +/** + * 这是一个可选的 Standard,用于协助第三方AppConn想要实现一些特殊的 Operation,这些Operation本身并不属于 + * 三大规范强制要求要实现的 Operation,但是会提供给三大规范或是DSS内嵌的应用工具使用。 + * @author enjoyyin + * @date 2022-03-18 + * @since 1.1.0 + */ +public class OptionalIntegrationStandard extends AbstractAppIntegrationStandard { + + private static Map optionalIntegrationStandards = new HashMap<>(); + private String appConnName; + + public static OptionalIntegrationStandard getInstance(String appConnName) { + if(!optionalIntegrationStandards.containsKey(appConnName)) { + synchronized (OptionalIntegrationStandard.class) { + if(!optionalIntegrationStandards.containsKey(appConnName)) { + OptionalIntegrationStandard standard = new OptionalIntegrationStandard(); + standard.setAppConnName(appConnName); + optionalIntegrationStandards.put(appConnName, standard); + } + } + } + return optionalIntegrationStandards.get(appConnName); + } + + public OptionalService getOptionalService(AppInstance appInstance) { + return getOrCreate(appInstance, OptionalService::new, OptionalService.class); + } + + public String getAppConnName() { + return appConnName; + } + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + @Override + protected void initService(T service) { + service.setAppStandard(this); + service.init(); + } + + @Override + public String getStandardName() { + return "OptionalIntegrationStandard"; + } + + @Override + public int getGrade() { + return 10; + } + + @Override + public void init() throws AppStandardErrorException { + + } + + @Override + public void close() { + + } +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureIntegrationStandard.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureIntegrationStandard.java new file mode 100644 index 000000000..40da6e061 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureIntegrationStandard.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure; + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.dss.standard.app.structure.role.RoleService; +import com.webank.wedatasphere.dss.standard.app.structure.status.AppStatusService; +import com.webank.wedatasphere.dss.standard.common.core.AppIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; + +/** + * DSS 的二级规范,为组织结构规范。组织结构规范主要提供了以下的能力: + * 1. 工程管理服务能力。工程管理服务用于打通 DSS 与第三方应用的工程体系。 + * 2. 角色管理服务能力。角色管理服务用于打通 DSS 与第三方应用的角色体系。 + * 3. 第三方应用状态管理能力。主要用于确认第三方应用的状态。 + *
+ * 建议用户直接继承 {@code AbstractStructureIntegrationStandard} + */ +public interface StructureIntegrationStandard extends AppIntegrationStandard { + + /** + * 统一角色规范,用于打通DSS与各集成接入系统的角色体系。 + * 该规范为预留规范,DSS 框架层暂未与这两个规范进行对接,用户直接返回 null 即可。 + * @param appInstance AppInstance 实例 + * @return 直接返回 null 即可 + */ + RoleService getRoleService(AppInstance appInstance); + + /** + * 统一工程规范,用于打通DSS与各集成接入系统的工程体系 + * @param appInstance AppInstance 实例 + * @return ProjectService 实现类 + */ + ProjectService getProjectService(AppInstance appInstance); + + /** + * 第三方应用状态检查规范。该规范为预留规范,DSS 框架层暂未与这两个规范进行对接,用户直接返回 null 即可。 + * @param appInstance AppInstance 实例 + * @return 直接返回 null 即可 + */ + AppStatusService getAppStateService(AppInstance appInstance); + + @Override + default String getStandardName() { + return "structureIntegrationStandard"; + } + + @Override + default int getGrade() { + return 2; + } + + @Override + default boolean isNecessary() { + return false; + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureOperation.java new file mode 100644 index 000000000..570d2eb90 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureOperation.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure; + + +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.service.Operation; + + +public interface StructureOperation + extends Operation { + + @Override + void init(); + + void setStructureService(StructureService service); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureRequestRef.java new file mode 100644 index 000000000..6c40b7877 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureRequestRef.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.standard.app.structure; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.ref.WorkspaceRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRefImpl; + +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-03-01 + * @since 0.5.0 + */ +public interface StructureRequestRef> extends WorkspaceRequestRef { + + R setName(String name); + + String getUserName(); + + R setUserName(String userName); + + R setDSSLabels(List dssLabels); + + R setType(String type); + + R setWorkspace(Workspace workspace); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureRequestRefImpl.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureRequestRefImpl.java new file mode 100644 index 000000000..aaf721e03 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureRequestRefImpl.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.standard.app.structure; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRefImpl; + +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-03-15 + * @since 0.5.0 + */ +public class StructureRequestRefImpl> + extends RequestRefImpl implements StructureRequestRef { + + private Workspace workspace; + private String userName; + + @Override + public R setName(String name) { + this.name = name; + return (R) this; + } + + @Override + public String getUserName() { + return userName; + } + + @Override + public R setUserName(String userName) { + this.userName = userName; + return (R) this; + } + + @Override + public R setDSSLabels(List dssLabels) { + this.dssLabels = dssLabels; + return (R) this; + } + + @Override + public R setType(String type) { + this.type = type; + return (R) this; + } + + @Override + public R setWorkspace(Workspace workspace) { + this.workspace = workspace; + return (R) this; + } + + @Override + public Workspace getWorkspace() { + return workspace; + } +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureService.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureService.java new file mode 100644 index 000000000..f8c8a99fe --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/StructureService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure; + + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppIntegrationService; + + +public interface StructureService extends AppIntegrationService { + + void setAppStandard(StructureIntegrationStandard appStandard); + + StructureIntegrationStandard getAppStandard(); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/AbstractOptionalOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/AbstractOptionalOperation.java new file mode 100644 index 000000000..74da86771 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/AbstractOptionalOperation.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.standard.app.structure.optional; + +import com.webank.wedatasphere.dss.standard.app.sso.operation.AbstractOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; + +/** + * @author enjoyyin + * @date 2022-03-18 + * @since 0.5.0 + */ +public abstract class AbstractOptionalOperation + extends AbstractOperation implements OptionalOperation { + + /** + * for more detail, please access the super explanation. + * @return null if you don't want to use SSORequestOperation; otherwise, a AppConn name is needed. + */ + @Override + protected String getAppConnName() { + return null; + } + + @Override + public final void setOptionalService(OptionalService optionalService) { + service = optionalService; + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/OptionalOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/OptionalOperation.java new file mode 100644 index 000000000..47f48d19a --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/OptionalOperation.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.standard.app.structure.optional; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.service.Operation; + +/** + * 这是一个可选的 Operation,用于协助第三方AppConn想要实现一些特殊的Operation,这些Operation本身并不属于 + * 三大规范强制要求要实现的 Operation,但是会提供给三大规范或是DSS内嵌的应用工具使用。 + * @author enjoyyin + * @date 2022-03-18 + * @since 1.1.0 + */ +public interface OptionalOperation extends Operation { + + String getOperationName(); + + void setOptionalService(OptionalService optionalService); + + V apply(K requestRef); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/OptionalService.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/OptionalService.java new file mode 100644 index 000000000..7b6fc3e66 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/optional/OptionalService.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.standard.app.structure.optional; + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.app.structure.OptionalIntegrationStandard; +import com.webank.wedatasphere.dss.standard.common.app.AppIntegrationServiceImpl; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.standard.common.utils.AppStandardClassUtils; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * 为第三方 AppConn 提供的通用 Service 服务,用于构建一些具备特殊能力的 Operation,以供DSS内嵌的应用工具使用。 + * @author enjoyyin + * @date 2022-03-18 + * @since 1.1.0 + */ +public class OptionalService extends AppIntegrationServiceImpl { + + private List operations; + private OptionalIntegrationStandard appStandard; + + public void init() { + operations = AppStandardClassUtils.getInstance(appStandard.getAppConnName()).getInstances(OptionalOperation.class); + operations.forEach(optionalOperation -> { + optionalOperation.setOptionalService(this); + optionalOperation.init(); + }); + LoggerFactory.getLogger(OptionalService.class).info("The AppConn {} initialized a list of OptionalOperations {}.", + getAppInstance().getBaseUrl(), operations.stream().map(OptionalOperation::getOperationName).collect(Collectors.joining(", "))); + } + + public OptionalOperation getOptionalOperation(String operationName) { + return operations.stream().filter(optionalOperation -> optionalOperation.getOperationName().equals(operationName)) + .findAny().orElseThrow(() -> new ExternalOperationFailedException(50322, "Not exists Optional Operation " + operationName)); + } + + public void setAppStandard(OptionalIntegrationStandard appStandard) { + this.appStandard = appStandard; + } + + public OptionalIntegrationStandard getAppStandard() { + return appStandard; + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectCreationOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectCreationOperation.java new file mode 100644 index 000000000..76ec81d0f --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectCreationOperation.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + + +public interface ProjectCreationOperation> + extends StructureOperation { + + /** + * Try to create the one-to-one related refProject in third-party AppConn. + * If created successfully, please return a ProjectResponseRef contained refProjectId, + * so DSS can use the refProjectId to operate the related refProject in third-party AppConn. + * The returned refProjectId is the other ProjectOperations which used. + *

+ * 该方法会尝试请求第三方应用工具创建一个与 DSS 工程一对一关联的第三方 refProject。 + * 如果创建成功,请返回一个包含了第三方应用工具的工程 ID(命名为 refProjectId)的 ProjectResponseRef, + * 以便 DSS 接下来可以使用 refProjectId 来管理第三方应用工具的这个工程(命名为 refProject)。 + *
+ * 返回的 refProjectId 是其他 ProjectOperation 能够操作这个第三方应用工具的工程的基础。DSS 在调用其他 + * ProjectOperation 时,会将该 refProjectId 作为方法参数传入,以便用户能正常找到对应的 refProject进行相应操作。 + * @param projectRef contains the DSS project info(包含了 DSS 的工程信息). + * @return a ProjectResponseRef contained refProjectId(返回一个包含了 refProjectId 的 ProjectResponseRef) + * @throws ExternalOperationFailedException 如果创建过程中发生异常,请抛出该异常。 + */ + ProjectResponseRef createProject(R projectRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectDeletionOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectDeletionOperation.java new file mode 100644 index 000000000..4ff0e0b28 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectDeletionOperation.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + + +public interface ProjectDeletionOperation> + extends StructureOperation { + + /** + * delete the related refProject in third-party AppConn by refProjectId + * which returned by ProjectCreationOperation. + * refProjectId must not be null, please use it to delete the refProject. + * @param projectRef refProject info, refProjectId must not be null + * @return the result of deletion, just success or failure. + * @throws ExternalOperationFailedException + */ + ResponseRef deleteProject(R projectRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectSearchOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectSearchOperation.java new file mode 100644 index 000000000..d03f3b61d --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectSearchOperation.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +public interface ProjectSearchOperation> + extends StructureOperation { + + /** + * Try to search the project by provide refProjectId or projectName. + * If refProjectId is not null, then the third-part AppConn should use refProjectId to search the refProject; + * otherwise, the third-part AppConn should use projectName to search the refProject. + *
+ * If the refProject is exists, please return a ProjectResponseRef which has been set in the refProjectId; + * otherwise, just return an empty succeeded ProjectResponseRef. + * @param projectRef the refProject info + * @return return a ProjectResponseRef contained refProjectId if the refProject is exists, otherwise + * just return an empty succeeded ProjectResponseRef. + * @throws ExternalOperationFailedException If some error are happened. + */ + ProjectResponseRef searchProject(R projectRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectService.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectService.java new file mode 100644 index 000000000..a1d7acf4e --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectService.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project; + +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureService; + +/** + * 实现了工程的统一创建、更新、删除和查重操作。 + * 用于打通 DSS 工程与接入的第三方应用工具的工程体系,实现工程的协同管理。 + */ +public abstract class ProjectService extends AbstractStructureService { + + /** + * This method is used to ensure if the third-part AppConn has the same permission + * management with DSS project or not. + *
+ * If permission management is exists and similar, please return true; otherwise + * please return false. + * @return return false by default. + */ + public boolean isCooperationSupported() { + return false; + } + + /** + * This is a pretty important method, please be aware of the instruction of this method. + * This method is used to ensure Whether the third-part AppConn allows duplicate project names. + * If this method returns false, which means duplicate project names is allowed, no exception will + * be throwed by DSS project framework when + * a new DSS project is creating and a duplicate project name check was not passed; otherwise, the + * project creation will be failed if a duplicate project name check was not passed. + *
+ * 这是一个非常重要的方法,请注意此方法的说明。 + * 此方法用于确保第三方 AppConn 是否允许重复的项目名称。 + * 如果该方法返回 false,即允许项目名重复,则在创建新的 DSS 项目并进行项目名重复性检查时, + * 即使第三方应用存在同名项目,DSS 框架也不会抛出异常; + * 否则,如果项目名重复性检查未通过,则 DSS 项目将创建失败。 + * @return return true by default, means if the third-part AppConn has exist a project with the same + * name, the DSS project creation will be failed(默认返回 true,表示如果第三方 AppConn 已经存在同名项目, + * 则 DSS 项目将创建失败). + */ + public boolean isProjectNameUnique() { + return true; + } + + public final ProjectCreationOperation getProjectCreationOperation() { + return getOrCreate(this::createProjectCreationOperation, ProjectCreationOperation.class); + } + + protected abstract ProjectCreationOperation createProjectCreationOperation(); + + public final ProjectUpdateOperation getProjectUpdateOperation() { + return getOrCreate(this::createProjectUpdateOperation, ProjectUpdateOperation.class); + } + + protected abstract ProjectUpdateOperation createProjectUpdateOperation(); + + public final ProjectDeletionOperation getProjectDeletionOperation() { + return getOrCreate(this::createProjectDeletionOperation, ProjectDeletionOperation.class); + } + + protected abstract ProjectDeletionOperation createProjectDeletionOperation(); + + + public final ProjectSearchOperation getProjectSearchOperation() { + return getOrCreate(this::createProjectSearchOperation, ProjectSearchOperation.class); + } + + protected abstract ProjectSearchOperation createProjectSearchOperation(); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectUpdateOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectUpdateOperation.java new file mode 100644 index 000000000..4d11d4d75 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ProjectUpdateOperation.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + + +public interface ProjectUpdateOperation> + extends StructureOperation { + + /** + * Try to update the related refProject in third-party AppConn. + * Usually, DSS only want to update the projectName, description and permissions info in third-party refProject, + * so the refProjectId is always exists and can not be changeable. + *
+ * Notice: do not try to change the refProjectId already related with the third-party refProject. + * @param projectRef contains the DSS project info updated. + * @return the result of update, just success or failure. + * @throws ExternalOperationFailedException + */ + ResponseRef updateProject(R projectRef) throws ExternalOperationFailedException; + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java new file mode 100644 index 000000000..64e68a2bc --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.standard.app.structure.project.ref; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; + +/** + * @author enjoyyin + * @date 2022-03-13 + * @since 0.5.0 + */ +public interface DSSProjectContentRequestRef> + extends StructureRequestRef { + + default DSSProject getDSSProject() { + return (DSSProject) this.getParameter("dssProject"); + } + + default R setDSSProject(DSSProject dssProject) { + setParameter("dssProject", dssProject); + return (R) this; + } + + /** + * DSS 工程的全量最新权限信息,包含了 DSS 工程所有的最新权限信息 + * @return DSSProjectPrivilege + */ + default DSSProjectPrivilege getDSSProjectPrivilege() { + return (DSSProjectPrivilege) this.getParameter("dssProjectPrivilege"); + } + + default R setDSSProjectPrivilege(DSSProjectPrivilege dssProjectPrivilege) { + setParameter("dssProjectPrivilege", dssProjectPrivilege); + return (R) this; + } + + class DSSProjectContentRequestRefImpl + extends StructureRequestRefImpl + implements DSSProjectContentRequestRef { + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectPrivilege.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectPrivilege.java new file mode 100644 index 000000000..952507b21 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectPrivilege.java @@ -0,0 +1,68 @@ +package com.webank.wedatasphere.dss.standard.app.structure.project.ref; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-04-19 + * @since 1.1.0 + */ +public interface DSSProjectPrivilege { + + DSSProjectPrivilege EMPTY = newBuilder().setAccessUsers(new ArrayList<>(0)) + .setEditUsers(new ArrayList<>(0)) + .setReleaseUsers(new ArrayList<>(0)).build(); + + List getAccessUsers(); + + List getEditUsers(); + + List getReleaseUsers(); + + static Builder newBuilder() { + return new Builder(); + } + + class Builder { + + protected List accessUsers; + protected List editUsers; + protected List releaseUsers; + + public Builder setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + return this; + } + + public Builder setEditUsers(List editUsers) { + this.editUsers = editUsers; + return this; + } + + public Builder setReleaseUsers(List releaseUsers) { + this.releaseUsers = releaseUsers; + return this; + } + + public DSSProjectPrivilege build() { + return new DSSProjectPrivilege(){ + @Override + public List getAccessUsers() { + return accessUsers; + } + + @Override + public List getEditUsers() { + return editUsers; + } + + @Override + public List getReleaseUsers() { + return releaseUsers; + } + }; + } + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/InternalProjectResponseRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/InternalProjectResponseRef.java new file mode 100644 index 000000000..06ac1e1a7 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/InternalProjectResponseRef.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.standard.app.structure.project.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.InternalResponseRef; + +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-03-08 + * @since 0.5.0 + */ +public class InternalProjectResponseRef extends InternalResponseRef implements ProjectResponseRef { + + private Long projectRefId; + + public InternalProjectResponseRef(String responseBody, int status, + String errorMsg, Map responseMap, + Long projectRefId) { + super(responseBody, status, errorMsg, responseMap); + this.projectRefId = projectRefId; + } + + @Override + public Long getRefProjectId() { + return projectRefId; + } +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectResponseRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectResponseRef.java new file mode 100644 index 000000000..11d939c36 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectResponseRef.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +import java.util.Map; + + +public interface ProjectResponseRef extends ResponseRef { + + Long getRefProjectId(); + + static InternalBuilder newInternalBuilder() { + return new InternalBuilder(); + } + + static ExternalBuilder newExternalBuilder() { + return new ExternalBuilder(); + } + + class InternalBuilder extends ResponseRefBuilder.InternalResponseRefBuilder { + protected Long refProjectId; + public InternalBuilder setRefProjectId(Long refProjectId) { + this.refProjectId = refProjectId; + return this; + } + @Override + protected InternalProjectResponseRef createResponseRef(String responseBody, int status, String errorMsg, Map responseMap) { + return new InternalProjectResponseRef(responseBody, status, errorMsg, responseMap, refProjectId); + } + } + + class ExternalBuilder extends ResponseRefBuilder.ExternalResponseRefBuilder { + protected Long refProjectId; + + public ExternalBuilder setRefProjectId(Long refProjectId) { + this.refProjectId = refProjectId; + return this; + } + + @Override + protected ProjectResponseRef createResponseRef() { + return new ProjectResponseRefImpl(); + } + + class ProjectResponseRefImpl extends ResponseRefImpl implements ProjectResponseRef { + public ProjectResponseRefImpl() { + super(ExternalBuilder.this.responseBody, ExternalBuilder.this.status, + ExternalBuilder.this.errorMsg, ExternalBuilder.this.responseMap); + } + @Override + public Long getRefProjectId() { + return refProjectId; + } + } + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java new file mode 100644 index 000000000..618b93edf --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.dss.standard.app.structure.project.ref; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; + +/** + * @author enjoyyin + * @date 2022-03-13 + * @since 1.1.0 + */ +public interface ProjectUpdateRequestRef> + extends DSSProjectContentRequestRef, RefProjectContentRequestRef { + + @Override + default String getProjectName() { + return getDSSProject().getName(); + } + + @Override + default R setProjectName(String projectName) { + getDSSProject().setName(projectName); + return (R) this; + } + + /** + * 只包含本次新增的 DSS 工程相关权限用户 + * @return + */ + default DSSProjectPrivilege getAddedDSSProjectPrivilege() { + return (DSSProjectPrivilege) this.getParameter("addedDSSProjectPrivilege"); + } + + default R setAddedDSSProjectPrivilege(DSSProjectPrivilege addedDSSProjectPrivilege) { + setParameter("addedDSSProjectPrivilege", addedDSSProjectPrivilege); + return (R) this; + } + + /** + * 只包含本次移除的 DSS 工程相关权限用户 + * @return + */ + default DSSProjectPrivilege getRemovedDSSProjectPrivilege() { + return (DSSProjectPrivilege) this.getParameter("removedDSSProjectPrivilege"); + } + + default R setRemovedDSSProjectPrivilege(DSSProjectPrivilege removedDSSProjectPrivilege) { + setParameter("removedDSSProjectPrivilege", removedDSSProjectPrivilege); + return (R) this; + } + + class ProjectUpdateRequestRefImpl extends StructureRequestRefImpl + implements ProjectUpdateRequestRef {} + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/RefProjectContentRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/RefProjectContentRequestRef.java new file mode 100644 index 000000000..5862f86a0 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/RefProjectContentRequestRef.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.standard.app.structure.project.ref; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; + +/** + * @author enjoyyin + * @date 2022-03-13 + * @since 0.5.0 + */ +public interface RefProjectContentRequestRef> + extends StructureRequestRef { + + default Long getRefProjectId() { + return (Long) getParameter("refProjectId"); + } + + default R setRefProjectId(Long refProjectId) { + setParameter("refProjectId", refProjectId); + return (R) this; + } + + default String getProjectName() { + return (String) getParameter("projectName"); + } + + default R setProjectName(String projectName) { + setParameter("projectName", projectName); + return (R) this; + } + + class RefProjectContentRequestRefImpl extends StructureRequestRefImpl + implements RefProjectContentRequestRef{} + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleCreationOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleCreationOperation.java new file mode 100644 index 000000000..1d14bdbb0 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleCreationOperation.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.role; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.role.ref.DSSRoleContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.role.ref.RoleResponseRef; + +/** + * 请求第三方系统创建一个与 DSS 角色一对一关联的第三方 refRole。 + */ +public interface RoleCreationOperation> + extends StructureOperation { + + /** + * 请求第三方系统创建一个与 DSS 角色一对一关联的第三方 refRole。 + * 考虑到不同的工作空间,其角色名有可能重复,因此建议第三方系统最好将角色名命名为 ${workspaceName}-${roleName}。 + * @param requestRef 包含了 DSS {@code Role} 信息的 RequestRef + * @return 包含了第三方系统的 Role Id 的 responseRef + */ + RoleResponseRef createRole(R requestRef); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleDeletionOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleDeletionOperation.java new file mode 100644 index 000000000..be1214f3f --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleDeletionOperation.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.role; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.role.ref.RefRoleContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; + +/** + * 请求第三方系统删除关联的第三方 refRole。 + * @param + */ +public interface RoleDeletionOperation> + extends StructureOperation { + + /** + * 请求第三方系统删除关联的第三方 refRole。 + * 删除时,请基于 requestRef.getRefRoleId() 进行删除。 + * @param requestRef 包含了 refRoleId 信息的 RequestRef + * @return 成功返回 ResponseRef.newExternalBuilder().success() 即可,失败请带上 error 信息 + */ + ResponseRef deleteRole(R requestRef); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleService.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleService.java new file mode 100644 index 000000000..962fe4857 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleService.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.role; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureService; + +/** + * 统一角色规范,用于打通DSS与各集成接入系统的角色体系。 + * 该规范为预留规范,DSS 框架层暂未与这两个规范进行对接。 + */ +public interface RoleService extends StructureService { + + /** + * 请求第三方系统创建一个与 DSS 角色一对一关联的第三方 refRole。 + * @return RoleCreationOperation 实现类 + */ + RoleCreationOperation getRoleCreationOperation(); + + /** + * 请求第三方系统更新与 DSS 角色已经一对一关联的第三方 refRole。 + * @return RoleUpdateOperation 实现类 + */ + RoleUpdateOperation getRoleUpdateOperation(); + + /** + * 请求第三方系统删除关联的第三方 refRole。 + * @return RoleDeletionOperation 实现类 + */ + RoleDeletionOperation getRoleDeletionOperation(); + + /** + * 用于当用户在 DSS 前端设置第三方系统的这个角色的权限信息时,返回第三方系统该角色的前端跳转页面 jumpURL。 + * @return RoleUrlOperation 实现类 + */ + RoleUrlOperation getRoleUrlOperation(); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleUpdateOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleUpdateOperation.java new file mode 100644 index 000000000..851c1a8d9 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleUpdateOperation.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.role; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.role.ref.RoleUpdateRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; + +/** + * 请求第三方系统更新与 DSS 角色已经一对一关联的第三方 refRole。 + */ +public interface RoleUpdateOperation> extends StructureOperation { + + /** + * 请求第三方系统更新与 DSS 角色已经一对一关联的第三方 refRole。 + * 考虑到不同的工作空间,其角色名有可能重复,因此建议第三方系统最好将角色名命名为 ${workspaceName}-${roleName}。 + * @param requestRef 包含了 DSS {@code Role} 修改后的信息的 RequestRef + * @return 成功返回 ResponseRef.newExternalBuilder().success() 即可,失败请带上 error 信息 + */ + ResponseRef updateRole(R requestRef); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleUrlOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleUrlOperation.java new file mode 100644 index 000000000..e5b57b920 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/RoleUrlOperation.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.role; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.role.ref.RefRoleContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.role.ref.RoleUrlResponseRef; + +/** + * 用于当用户在 DSS 前端设置第三方系统的这个角色的权限信息时,返回第三方系统该角色的前端跳转页面 jumpURL。 + * 该功能主要是为了尽量避免用户需要跳转多个系统去设置角色权限所预留的。 + *
+ * 设计的初衷是尽量不干扰第三方系统的角色权限体系,只是通过 Iframe 嵌入的方式,方便用户在一个页面即可完成所有角色 + * 权限信息的设置。 + */ +public interface RoleUrlOperation> + extends StructureOperation { + + /** + * 用于当用户在 DSS 前端设置第三方系统的这个角色的权限信息时,返回第三方系统该角色的前端跳转页面 jumpURL。 + * @param requestRef 带有 refRoleId 信息的 RefRoleContentRequestRef + * @return 请返回带有 jumpURL 的 RoleUrlResponseRef + */ + RoleUrlResponseRef getRoleUrl(R requestRef); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/DSSRoleContentRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/DSSRoleContentRequestRef.java new file mode 100644 index 000000000..c65863d30 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/DSSRoleContentRequestRef.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.standard.app.structure.role.ref; + +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.ref.WorkspaceRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; + +/** + * @author enjoyyin + * @date 2022-05-07 + * @since 1.1.0 + */ +public interface DSSRoleContentRequestRef> + extends RoleRequestRef { + + /** + * 包含了 DSS {@code Role} 的信息 + * @return Role + */ + default Role getRole() { + return (Role) getParameter("role"); + } + + default R setRole(Role role) { + setParameter("role", role); + return (R) this; + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RefRoleContentRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RefRoleContentRequestRef.java new file mode 100644 index 000000000..9d52b9d9b --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RefRoleContentRequestRef.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.standard.app.structure.role.ref; + +/** + * @author enjoyyin + * @since 1.1.0 + */ +public interface RefRoleContentRequestRef> + extends RoleRequestRef { + + default Long getRefRoleId() { + return (Long) getParameter("refRoleId"); + } + + default R setRefRoleId(Long refRoleId) { + setParameter("refRoleId", refRoleId); + return (R) this; + } + + default String getRoleName() { + return (String) getParameter("roleName"); + } + + default R setRoleName(String roleName) { + setParameter("roleName", roleName); + return (R) this; + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/Role.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/Role.java new file mode 100644 index 000000000..dc7db944b --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/Role.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.role.ref; + +import java.util.Date; +import java.util.List; + + +public interface Role { + + /** + * 角色英文名,每个工作空间唯一 + * @return 角色英文名 + */ + String getName(); + + /** + * 角色中文名,如果不存在则与 {@code getName()} 完全相同。 + * @return 角色中文名 + */ + String getShowName(); + + Date getCreateTime(); + + String getCreator(); + + String getDescription(); + + String getUpdator(); + + String getLastUpdateTime(); + + /** + * 该角色目前已绑定的所有用户列表 + * @return + */ + List getUserNames(); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleRequestRef.java new file mode 100644 index 000000000..8a46e3dbc --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleRequestRef.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.standard.app.structure.role.ref; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; + +/** + * RoleRequestRef 为 RoleOperation 的 RequestRef 基类。 + * 主要提供了各个 RoleRequestRef 子接口的实现类以供使用。 + * @author enjoyyin + * @since 1.1.0 + */ +public interface RoleRequestRef> extends StructureRequestRef { + + class DSSRoleContentRequestRefImpl extends StructureRequestRefImpl + implements DSSRoleContentRequestRef{} + + class RefRoleContentRequestRefImpl extends StructureRequestRefImpl + implements RefRoleContentRequestRef {} + + class RoleUpdateRequestRefImpl extends StructureRequestRefImpl + implements RoleUpdateRequestRef {} + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleResponseRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleResponseRef.java new file mode 100644 index 000000000..711984fbd --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleResponseRef.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.role.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +public interface RoleResponseRef extends ResponseRef { + + Long getRefRoleId(); + + class Builder extends ResponseRefBuilder.ExternalResponseRefBuilder { + protected Long refRoleId; + + public Builder setRefRoleId(Long refRoleId) { + this.refRoleId = refRoleId; + return this; + } + + @Override + protected RoleResponseRef createResponseRef() { + return new RoleResponseRefImpl(); + } + + class RoleResponseRefImpl extends ResponseRefImpl implements RoleResponseRef { + public RoleResponseRefImpl() { + super(Builder.this.responseBody, Builder.this.status, + Builder.this.errorMsg, Builder.this.responseMap); + } + @Override + public Long getRefRoleId() { + return refRoleId; + } + } + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleUpdateRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleUpdateRequestRef.java new file mode 100644 index 000000000..cf1da4172 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleUpdateRequestRef.java @@ -0,0 +1,60 @@ +package com.webank.wedatasphere.dss.standard.app.structure.role.ref; + +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author enjoyyin + * @since 1.1.0 + */ +public interface RoleUpdateRequestRef> extends + DSSRoleContentRequestRef, RefRoleContentRequestRef { + + @Override + default String getRoleName() { + if(getRole() == null) { + return null; + } + return getRole().getName(); + } + + @Override + default R setRoleName(String roleName) { + throw new ExternalOperationWarnException(90030, "not support method."); + } + + /** + * 该角色本次新增的所有用户列表 + * @return 本次新增的所有用户列表 + */ + default List getAddedUserNames() { + if(!getParameters().containsKey("addedUserNames")) { + return new ArrayList<>(); + } + return (List) getParameter("addedUserNames"); + } + + default R setAddedUserNames(List addedUserNames) { + setParameter("addedUserNames", addedUserNames); + return (R) this; + } + + /** + * 该角色本次删除的所有用户列表 + * @return 本次新增的所有用户列表 + */ + default List getRemovedUserNames() { + if(!getParameters().containsKey("removedUserNames")) { + return new ArrayList<>(); + } + return (List) getParameter("removedUserNames"); + } + + default R setRemovedUserNames(List removedUserNames) { + setParameter("removedUserNames", removedUserNames); + return (R) this; + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleUrlResponseRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleUrlResponseRef.java new file mode 100644 index 000000000..10d6d2e6f --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/role/ref/RoleUrlResponseRef.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.dss.standard.app.structure.role.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +/** + * @author enjoyyin + * @date 2022-05-07 + * @since 1.1.0 + */ +public interface RoleUrlResponseRef extends ResponseRef { + + String getRoleJumpUrl(); + + static Builder newBuilder() { + return new Builder(); + } + + class Builder extends ResponseRefBuilder.ExternalResponseRefBuilder { + protected String roleJumpUrl; + + public Builder setRoleJumpUrl(String roleJumpUrl) { + this.roleJumpUrl = roleJumpUrl; + return this; + } + + @Override + protected RoleUrlResponseRef createResponseRef() { + return new RoleUrlResponseRefImpl(); + } + + class RoleUrlResponseRefImpl extends ResponseRefImpl implements RoleUrlResponseRef { + public RoleUrlResponseRefImpl() { + super(Builder.this.responseBody, Builder.this.status, + Builder.this.errorMsg, Builder.this.responseMap); + } + + @Override + public String getRoleJumpUrl() { + return roleJumpUrl; + } + } + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatus.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatus.java new file mode 100644 index 000000000..b1d25c89d --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatus.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.status; + +public interface AppStatus { + + /** + * 返回第三方系统的状态,其中:0 代表健康,为绿色;1 代表繁忙,为黄色;2 代表响应慢,为橙色;3 代表失联,为灰色 + * @return 返回状态 + */ + int getStatus(); + + /** + * 当 {@code getStatus()} 非 0 时,请返回诊断信息 + * @return 诊断信息 + */ + String getUnHealthyMsg(); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusOperation.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusOperation.java new file mode 100644 index 000000000..2e6085329 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusOperation.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.status; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; + +/** + * 检查第三方系统的状态,如果发现该第三方系统出现异常,以便及时告警或展示给到前端使用用户。 + */ +public interface AppStatusOperation + extends StructureOperation { + + /** + * 无入参,仅用于检查第三方系统的状态。 + * @return 返回心跳结果 + */ + AppStatusResponseRef heartbeat(); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusResponseRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusResponseRef.java new file mode 100644 index 000000000..97f113fb5 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusResponseRef.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.status; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefBuilder; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl; + +public interface AppStatusResponseRef extends ResponseRef { + + AppStatus getAppStatus(); + + static Builder newBuilder() { + return new Builder(); + } + + class Builder extends ResponseRefBuilder.ExternalResponseRefBuilder { + protected AppStatus appStatus; + + public Builder setAppStatus(AppStatus appStatus) { + this.appStatus = appStatus; + return this; + } + + @Override + protected AppStatusResponseRef createResponseRef() { + return new AppStatusResponseRefImpl(); + } + + class AppStatusResponseRefImpl extends ResponseRefImpl implements AppStatusResponseRef { + public AppStatusResponseRefImpl() { + super(Builder.this.responseBody, Builder.this.status, + Builder.this.errorMsg, Builder.this.responseMap); + } + + @Override + public AppStatus getAppStatus() { + return appStatus; + } + } + } + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusService.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusService.java new file mode 100644 index 000000000..62ba31308 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/status/AppStatusService.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.status; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureService; + +/** + * 第三方应用状态检查规范。 + */ +public interface AppStatusService extends StructureService { + + /** + * 返回第三方应用状态检查操作。 + * @return AppStatusOperation 实现类 + */ + AppStatusOperation getAppStatusOperation(); + +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/utils/StructureOperationUtils.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/utils/StructureOperationUtils.java new file mode 100644 index 000000000..746102b83 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/utils/StructureOperationUtils.java @@ -0,0 +1,75 @@ +package com.webank.wedatasphere.dss.standard.app.structure.utils; + +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.standard.app.structure.StructureOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.StructureService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardWarnException; +import com.webank.wedatasphere.dss.standard.common.utils.RequestRefUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * @author enjoyyin + * @date 2022-03-14 + * @since 0.5.0 + */ +public class StructureOperationUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(StructureOperationUtils.class); + + public static V tryStructureOperation(Supplier getStructureService, + Function getStructureOperation, + Consumer structureRequestRefConsumer, + BiFunction responseRefConsumer, + String errorMsg) { + S structureService = getStructureService.get(); + if(structureService == null) { + LOGGER.warn("{} is ignored. Caused by: The AppConn has no StructureService.", errorMsg); + return null; + } + StructureOperation operation = getStructureOperation.apply(structureService); + if(operation == null) { + LOGGER.warn("{} is ignored. Caused by: The corresponding operation is not exists.", errorMsg); + return null; + } + K requestRef = RequestRefUtils.getRequestRef(operation); + if(structureRequestRefConsumer != null) { + structureRequestRefConsumer.accept(requestRef); + } + V responseRef = responseRefConsumer.apply(operation, requestRef); + if(responseRef.isFailed()) { + LOGGER.error("{} failed. Caused by: {}.", errorMsg, responseRef.getErrorMsg()); + DSSExceptionUtils.dealWarnException(61123, + String.format("%s failed. Caused by: %s.", errorMsg, responseRef.getErrorMsg()), + AppStandardWarnException.class); + } + return responseRef; + } + + public static V tryProjectOperation(Supplier getProjectService, + Function getProjectOperation, + Consumer dssProjectContentRequestRefConsumer, + Consumer refProjectContentRequestRefConsumer, + BiFunction responseRefConsumer, + String errorMsg) { + return tryStructureOperation(getProjectService, getProjectOperation, structureRequestRef -> { + if(dssProjectContentRequestRefConsumer != null && structureRequestRef instanceof DSSProjectContentRequestRef) { + dssProjectContentRequestRefConsumer.accept((DSSProjectContentRequestRef) structureRequestRef); + } + if(refProjectContentRequestRefConsumer != null && structureRequestRef instanceof RefProjectContentRequestRef) { + refProjectContentRequestRefConsumer.accept((RefProjectContentRequestRef) structureRequestRef); + } + }, responseRefConsumer, errorMsg); + } + +} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml b/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml new file mode 100644 index 000000000..f41806fda --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml @@ -0,0 +1,84 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../../pom.xml + + 4.0.0 + + spring-origin-dss-project-plugin + + + + com.webank.wedatasphere.dss + dss-project-plugin + ${dss.version} + + + org.springframework + spring-core + ${spring.version} + compile + + + org.springframework + spring-context + ${spring.version} + compile + + + org.springframework + spring-web + ${spring.version} + compile + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + + + \ No newline at end of file diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectCooperationSpringFilter.java b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectCooperationSpringFilter.java new file mode 100644 index 000000000..447be5a61 --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/filter/ProjectCooperationSpringFilter.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.filter; + +import com.webank.wedatasphere.dss.standard.app.sso.plugin.filter.UserInterceptor; +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.ProjectCooperationPlugin; +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin.OriginProjectCooperationPlugin; +import javax.servlet.FilterConfig; +import org.springframework.stereotype.Component; +import org.springframework.web.context.WebApplicationContext; +import org.springframework.web.context.support.WebApplicationContextUtils; + + +@Component +public class ProjectCooperationSpringFilter extends ProjectCooperationFilter { + + @Override + public ProjectAuthInterceptor getProjectAuthInterceptor(FilterConfig filterConfig) { + WebApplicationContext webApplicationContext = + WebApplicationContextUtils.getRequiredWebApplicationContext(filterConfig.getServletContext()); + return webApplicationContext.getBean(ProjectAuthInterceptor.class); + } + + @Override + public UserInterceptor getUserAuthInterceptor(FilterConfig filterConfig) { + WebApplicationContext webApplicationContext = + WebApplicationContextUtils.getRequiredWebApplicationContext(filterConfig.getServletContext()); + return webApplicationContext.getBean(UserInterceptor.class); + } + + @Override + public ProjectCooperationPlugin getProjectCooperationPlugin(FilterConfig filterConfig) { + return OriginProjectCooperationPlugin.getProjectCooperatePlugin(); + } +} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/OriginProjectCooperationPlugin.scala b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/OriginProjectCooperationPlugin.scala new file mode 100644 index 000000000..5992ccf07 --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/OriginProjectCooperationPlugin.scala @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard +import com.webank.wedatasphere.dss.standard.app.sso.builder.DssMsgBuilderOperation.DSSMsg +import com.webank.wedatasphere.dss.standard.app.sso.origin.OriginSSOIntegrationStandardFactory +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.{ProjectAuth, ProjectCooperationPlugin} +import org.apache.linkis.httpclient.request.HttpAction +import javax.servlet.http.HttpServletRequest + + +class OriginProjectCooperationPlugin private() extends ProjectCooperationPlugin { + private val ssoIntegrationStandard:SSOIntegrationStandard =new OriginSSOIntegrationStandardFactory().getSSOIntegrationStandard + + private def getProjectAuth(action: HttpAction, dssMsg: DSSMsg): ProjectAuthImpl = { + val dwsHttpClient = HttpClient.getDSSClient(dssMsg.getDSSUrl) + val projectAuth = new ProjectAuthImpl + projectAuth.setWorkspaceName(dssMsg.getWorkspaceName) + HttpClient.addCookies(dssMsg, action) + dwsHttpClient.execute(action) match { + case projectAuthResult: ProjectAuthResult => + projectAuth.setProjectId(projectAuthResult.getProjectId) + projectAuth.setProjectName(projectAuthResult.getProjectName) + projectAuth.setEditUsers(projectAuthResult.getEditUsers) + projectAuth.setAccessUsers(projectAuthResult.getAccessUsers) + projectAuth.setDeleteUsers(projectAuthResult.getDeleteUsers) + } + projectAuth + } + + override def getProjectAuth(request: HttpServletRequest, projectId: String): ProjectAuth = { + val dssMsg = ssoIntegrationStandard.getSSOPluginService + .createDssMsgCacheOperation().getDSSMsgInSession(request) + val projectAuthAction = new ProjectAuthByIdAction + projectAuthAction.setWorkspace(dssMsg.getWorkspaceName) + projectAuthAction.setProjectId(projectId) + projectAuthAction.setComponentName(dssMsg.getAppName) + val projectAuth = getProjectAuth(projectAuthAction, dssMsg) + projectAuth.setProjectId(projectId) + projectAuth + } + + override def getProjectAuthByName(request: HttpServletRequest, projectName: String): ProjectAuth = { + + val dssMsg = ssoIntegrationStandard.getSSOPluginService + .createDssMsgCacheOperation().getDSSMsgInSession(request) + val projectAuthAction = new ProjectAuthByNameAction + projectAuthAction.setProjectName(projectName) + projectAuthAction.setWorkspace(dssMsg.getWorkspaceName) + val projectAuth = getProjectAuth(projectAuthAction, dssMsg) + projectAuth.setProjectName(projectName) + projectAuth + } +} +object OriginProjectCooperationPlugin { + private val projectCooperatePlugin = new OriginProjectCooperationPlugin + def getProjectCooperatePlugin: ProjectCooperationPlugin = projectCooperatePlugin +} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/OriginProjectPlugin.scala b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/OriginProjectPlugin.scala new file mode 100644 index 000000000..8e23ada1d --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/OriginProjectPlugin.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin + +import java.util + +import com.webank.wedatasphere.dss.standard.app.sso.SSOIntegrationStandard +import com.webank.wedatasphere.dss.standard.app.sso.origin.OriginSSOIntegrationStandardFactory +import com.webank.wedatasphere.dss.standard.app.sso.origin.client.HttpClient +import com.webank.wedatasphere.dss.standard.app.structure.project.plugin.ProjectPlugin +import javax.servlet.http.HttpServletRequest + + +class OriginProjectPlugin private() extends ProjectPlugin { + + override def getProjects(request: HttpServletRequest): util.List[String] = { + val ssoIntegrationStandard:SSOIntegrationStandard =new OriginSSOIntegrationStandardFactory().getSSOIntegrationStandard + val dssMsg =ssoIntegrationStandard.getSSOPluginService.createDssMsgCacheOperation().getDSSMsgInSession(request) + val dwsHttpClient = HttpClient.getDSSClient(dssMsg.getDSSUrl) + val projectListAction = new ProjectListByWorkspaceAction + projectListAction.setWorkspace(dssMsg.getWorkspaceName) + HttpClient.addCookies(dssMsg, projectListAction) + dwsHttpClient.execute(projectListAction) match { + case projectListResult: ProjectListByWorkspaceResult => + projectListResult.getProjectIds + } + } + +} +object OriginProjectPlugin { + private val projectPlugin = new OriginProjectPlugin + def getProjectPlugin: ProjectPlugin = projectPlugin +} \ No newline at end of file diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthByIdAction.scala b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthByIdAction.scala new file mode 100644 index 000000000..17596a6c7 --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthByIdAction.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.POSTAction + + +class ProjectAuthByIdAction extends POSTAction with DWSHttpAction { + + override def suffixURLs: Array[String] = Array("dss", "getProjectAuthOfWorkspace") + + def setWorkspace(workspaceName: String): Unit = addRequestPayload("workspaceName", workspaceName) + + def setProjectId(projectId: String): Unit = addRequestPayload("projectId", projectId) + + def setComponentName(componentName: String): Unit = addRequestPayload("componentName", componentName) + + override def getRequestPayload: String = "" +} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthByNameAction.scala b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthByNameAction.scala new file mode 100644 index 000000000..589af4342 --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthByNameAction.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.POSTAction + + +class ProjectAuthByNameAction extends POSTAction with DWSHttpAction { + + override def suffixURLs: Array[String] = Array("dss", "getProjectAuthOfWorkspace") + + def setWorkspace(workspaceName: String): Unit = addRequestPayload("workspaceName", workspaceName) + + def setProjectName(projectName: String): Unit = addRequestPayload("projectName", projectName) + + override def getRequestPayload: String = "" +} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthResult.scala b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthResult.scala new file mode 100644 index 000000000..7acf6c5ca --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectAuthResult.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin + +import scala.beans.BeanProperty +import java.util + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/getProjectAuthOfWorkspace") +class ProjectAuthResult extends DWSResult { + + @BeanProperty + var projectName: String = _ + + @BeanProperty + var projectId: String = _ + + @BeanProperty + var workspaceName: String = _ + + @BeanProperty + var editUsers: util.List[String] = _ + + @BeanProperty + var accessUsers: util.List[String] = _ + + @BeanProperty + var deleteUsers: util.List[String] = _ + +} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectListByWorkspaceAction.scala b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectListByWorkspaceAction.scala new file mode 100644 index 000000000..df46050a1 --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectListByWorkspaceAction.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.POSTAction + + +class ProjectListByWorkspaceAction extends POSTAction with DWSHttpAction { + + override def suffixURLs: Array[String] = Array("dss", "getProjectListOfWorkspace") + + def setWorkspace(workspaceName: String): Unit = addRequestPayload("workspaceName", workspaceName) + + override def getRequestPayload: String = "" +} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectListByWorkspaceResult.scala b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectListByWorkspaceResult.scala new file mode 100644 index 000000000..f8f4aef0b --- /dev/null +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/src/main/scala/com/webank/wedatasphere/dss/standard/app/structure/project/plugin/origin/ProjectListByWorkspaceResult.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.standard.app.structure.project.plugin.origin + +import java.util + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/getProjectListOfWorkspace") +class ProjectListByWorkspaceResult extends DWSResult { + + @BeanProperty + var projectIds: util.List[String] = _ + +} diff --git a/images/en_US/readme/DSS.png b/images/en_US/readme/DSS.png deleted file mode 100644 index 238b99607..000000000 Binary files a/images/en_US/readme/DSS.png and /dev/null differ diff --git a/images/en_US/readme/DSS_logo.png b/images/en_US/readme/DSS_logo.png new file mode 100644 index 000000000..8ca80f95d Binary files /dev/null and b/images/en_US/readme/DSS_logo.png differ diff --git a/images/zh_CN/charpter3/manual/components.png b/images/zh_CN/charpter3/manual/components.png deleted file mode 100644 index bc5612f56..000000000 Binary files a/images/zh_CN/charpter3/manual/components.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/components02.png b/images/zh_CN/charpter3/manual/components02.png deleted file mode 100644 index a58185ed4..000000000 Binary files a/images/zh_CN/charpter3/manual/components02.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/connector.png b/images/zh_CN/charpter3/manual/connector.png deleted file mode 100644 index 0d51ab482..000000000 Binary files a/images/zh_CN/charpter3/manual/connector.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/functions.png b/images/zh_CN/charpter3/manual/functions.png deleted file mode 100644 index 677bc552d..000000000 Binary files a/images/zh_CN/charpter3/manual/functions.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/project.png b/images/zh_CN/charpter3/manual/project.png deleted file mode 100644 index 86866b2d7..000000000 Binary files a/images/zh_CN/charpter3/manual/project.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/project01.png b/images/zh_CN/charpter3/manual/project01.png deleted file mode 100644 index abbeb18dd..000000000 Binary files a/images/zh_CN/charpter3/manual/project01.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/project02.png b/images/zh_CN/charpter3/manual/project02.png deleted file mode 100644 index cc7ac6a54..000000000 Binary files a/images/zh_CN/charpter3/manual/project02.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/sendemail01.png b/images/zh_CN/charpter3/manual/sendemail01.png deleted file mode 100644 index ebd319a78..000000000 Binary files a/images/zh_CN/charpter3/manual/sendemail01.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/signal02.png b/images/zh_CN/charpter3/manual/signal02.png deleted file mode 100644 index 7386576fc..000000000 Binary files a/images/zh_CN/charpter3/manual/signal02.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/signal03.png b/images/zh_CN/charpter3/manual/signal03.png deleted file mode 100644 index 42479659d..000000000 Binary files a/images/zh_CN/charpter3/manual/signal03.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/signal04.png b/images/zh_CN/charpter3/manual/signal04.png deleted file mode 100644 index f02df77aa..000000000 Binary files a/images/zh_CN/charpter3/manual/signal04.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/signal_01.png b/images/zh_CN/charpter3/manual/signal_01.png deleted file mode 100644 index 20936d9f1..000000000 Binary files a/images/zh_CN/charpter3/manual/signal_01.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/subflow01.png b/images/zh_CN/charpter3/manual/subflow01.png deleted file mode 100644 index ef7b6689f..000000000 Binary files a/images/zh_CN/charpter3/manual/subflow01.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow01.png b/images/zh_CN/charpter3/manual/workflow01.png deleted file mode 100644 index 736cdadb0..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow01.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow02.png b/images/zh_CN/charpter3/manual/workflow02.png deleted file mode 100644 index 1acc1b611..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow02.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow03.png b/images/zh_CN/charpter3/manual/workflow03.png deleted file mode 100644 index a4dd43695..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow03.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow04.png b/images/zh_CN/charpter3/manual/workflow04.png deleted file mode 100644 index d1f22bdae..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow04.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow05.png b/images/zh_CN/charpter3/manual/workflow05.png deleted file mode 100644 index fa22bd5f5..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow05.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow06.png b/images/zh_CN/charpter3/manual/workflow06.png deleted file mode 100644 index 095d6e2cc..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow06.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow07.png b/images/zh_CN/charpter3/manual/workflow07.png deleted file mode 100644 index 7c02b6936..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow07.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow08.png b/images/zh_CN/charpter3/manual/workflow08.png deleted file mode 100644 index 66d1efa9e..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow08.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow10.png b/images/zh_CN/charpter3/manual/workflow10.png deleted file mode 100644 index 0e8783394..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow10.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow11.png b/images/zh_CN/charpter3/manual/workflow11.png deleted file mode 100644 index 7ebe6f70c..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow11.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow12.png b/images/zh_CN/charpter3/manual/workflow12.png deleted file mode 100644 index 640c1640b..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow12.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow13.png b/images/zh_CN/charpter3/manual/workflow13.png deleted file mode 100644 index c152f2634..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow13.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow14.png b/images/zh_CN/charpter3/manual/workflow14.png deleted file mode 100644 index 052073bfe..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow14.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow15.png b/images/zh_CN/charpter3/manual/workflow15.png deleted file mode 100644 index ab0439005..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow15.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow16.png b/images/zh_CN/charpter3/manual/workflow16.png deleted file mode 100644 index 2661bacf0..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow16.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow17.png b/images/zh_CN/charpter3/manual/workflow17.png deleted file mode 100644 index 3e1ddb7cf..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow17.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow18.png b/images/zh_CN/charpter3/manual/workflow18.png deleted file mode 100644 index b2621c874..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow18.png and /dev/null differ diff --git a/images/zh_CN/charpter3/manual/workflow_09.png b/images/zh_CN/charpter3/manual/workflow_09.png deleted file mode 100644 index 3fdd8322e..000000000 Binary files a/images/zh_CN/charpter3/manual/workflow_09.png and /dev/null differ diff --git a/images/zh_CN/readme/architecture.png b/images/zh_CN/readme/architecture.png index 8195dc7da..348746c39 100644 Binary files a/images/zh_CN/readme/architecture.png and b/images/zh_CN/readme/architecture.png differ diff --git a/k8s/build.info b/k8s/build.info new file mode 100644 index 000000000..6d2222074 --- /dev/null +++ b/k8s/build.info @@ -0,0 +1,8 @@ +dss-flow-execution-server +dss-apiservice-server +dss-framework-orchestrator-server +dss-framework-project-server +dss-workflow-server +dss-data-api-server +dss-data-governance-server +dss-guide-server diff --git a/k8s/build.sh b/k8s/build.sh new file mode 100644 index 000000000..d2d86dcaa --- /dev/null +++ b/k8s/build.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +if [ -n "$1" ]; then + echo 'start building images' + docker pull base:1.0.0 + cat k8s/build.info | while read line; do + { + imageName="${line}:$1" + echo "build image: ${imageName}" + + docker build -t $imageName -f k8s/dockerfile/${line}.Dockerfile assembly/dss-package/target/out/dss-1.0.1 + docker push ${imageName} + } + done + echo 'build finished' +else + echo "Usage: sh build.sh {versionNumber}" +fi \ No newline at end of file diff --git a/k8s/dockerfile/dss-apiservice-server.Dockerfile b/k8s/dockerfile/dss-apiservice-server.Dockerfile new file mode 100644 index 000000000..ab431a5b3 --- /dev/null +++ b/k8s/dockerfile/dss-apiservice-server.Dockerfile @@ -0,0 +1,9 @@ +FROM base:1.0.0 + +WORKDIR /opt/dss + +COPY lib/dss-commons /opt/dss/dss-commons/ +COPY lib/dss-apps/dss-apiservice-server /opt/dss/dss-apiservice-server/lib/ +COPY sbin/k8s/dss-apiservice-server.sh /opt/dss/dss-apiservice-server/bin/startup.sh + +ENTRYPOINT ["bash","dss-apiservice-server/bin/startup.sh"] \ No newline at end of file diff --git a/k8s/dockerfile/dss-data-api-server.Dockerfile b/k8s/dockerfile/dss-data-api-server.Dockerfile new file mode 100644 index 000000000..ed3d5c2c5 --- /dev/null +++ b/k8s/dockerfile/dss-data-api-server.Dockerfile @@ -0,0 +1,9 @@ +FROM base:1.0.0 + +WORKDIR /opt/dss + +COPY lib/dss-commons /opt/dss/dss-commons/ +COPY lib/dss-data-api/dss-data-api-server /opt/dss/dss-data-api-server/lib/ +COPY sbin/k8s/dss-data-api-server.sh /opt/dss/dss-data-api-server/bin/startup.sh + +ENTRYPOINT ["bash","dss-data-api-server/bin/startup.sh"] \ No newline at end of file diff --git a/k8s/dockerfile/dss-data-governance-server.Dockerfile b/k8s/dockerfile/dss-data-governance-server.Dockerfile new file mode 100644 index 000000000..0c7144921 --- /dev/null +++ b/k8s/dockerfile/dss-data-governance-server.Dockerfile @@ -0,0 +1,9 @@ +FROM base:1.0.0 + +WORKDIR /opt/dss + +COPY lib/dss-commons /opt/dss/dss-commons/ +COPY lib/dss-data-governance/dss-data-governance-server /opt/dss/dss-data-governance-server/lib/ +COPY sbin/k8s/dss-data-governance-server.sh /opt/dss/dss-data-governance-server/bin/startup.sh + +ENTRYPOINT ["bash","dss-data-governance-server/bin/startup.sh"] \ No newline at end of file diff --git a/k8s/dockerfile/dss-flow-execution-server.Dockerfile b/k8s/dockerfile/dss-flow-execution-server.Dockerfile new file mode 100644 index 000000000..0c04f8768 --- /dev/null +++ b/k8s/dockerfile/dss-flow-execution-server.Dockerfile @@ -0,0 +1,9 @@ +FROM base:1.0.0 + +WORKDIR /opt/dss + +COPY lib/dss-commons /opt/dss/dss-commons/ +COPY lib/dss-orchestrator/dss-flow-execution-server /opt/dss/dss-flow-execution-server/lib/ +COPY sbin/k8s/dss-flow-execution-server.sh /opt/dss/dss-flow-execution-server/bin/startup.sh + +ENTRYPOINT ["bash","dss-flow-execution-server/bin/startup.sh"] \ No newline at end of file diff --git a/k8s/dockerfile/dss-framework-orchestrator-server.Dockerfile b/k8s/dockerfile/dss-framework-orchestrator-server.Dockerfile new file mode 100644 index 000000000..089c54b15 --- /dev/null +++ b/k8s/dockerfile/dss-framework-orchestrator-server.Dockerfile @@ -0,0 +1,10 @@ +FROM base:1.0.0 + +WORKDIR /opt/dss + +COPY lib/dss-commons /opt/dss/dss-commons/ +COPY dss-appconns /opt/dss/dss-appconns +COPY lib/dss-framework/dss-framework-orchestrator-server /opt/dss/dss-framework-orchestrator-server/lib/ +COPY sbin/k8s/dss-framework-orchestrator-server.sh /opt/dss/dss-framework-orchestrator-server/bin/startup.sh + +ENTRYPOINT ["bash","dss-framework-orchestrator-server/bin/startup.sh"] \ No newline at end of file diff --git a/k8s/dockerfile/dss-framework-project-server.Dockerfile b/k8s/dockerfile/dss-framework-project-server.Dockerfile new file mode 100644 index 000000000..23d1ca77c --- /dev/null +++ b/k8s/dockerfile/dss-framework-project-server.Dockerfile @@ -0,0 +1,10 @@ +FROM base:1.0.0 + +WORKDIR /opt/dss + +COPY lib/dss-commons /opt/dss/dss-commons/ +COPY dss-appconns /opt/dss/dss-appconns +COPY lib/dss-framework/dss-framework-project-server /opt/dss/dss-framework-project-server/lib/ +COPY sbin/k8s/dss-framework-project-server.sh /opt/dss/dss-framework-project-server/bin/startup.sh + +ENTRYPOINT ["bash","dss-framework-project-server/bin/startup.sh"] \ No newline at end of file diff --git a/k8s/dockerfile/dss-guide-server.Dockerfile b/k8s/dockerfile/dss-guide-server.Dockerfile new file mode 100644 index 000000000..81813601d --- /dev/null +++ b/k8s/dockerfile/dss-guide-server.Dockerfile @@ -0,0 +1,9 @@ +FROM base:1.0.0 + +WORKDIR /opt/dss + +COPY lib/dss-commons /opt/dss/dss-commons/ +COPY lib/dss-guide/dss-guide-server /opt/dss/dss-guide-server/lib/ +COPY sbin/k8s/dss-guide-server.sh /opt/dss/dss-guide-server/bin/startup.sh + +ENTRYPOINT ["bash","dss-guide-server/bin/startup.sh"] \ No newline at end of file diff --git a/k8s/dockerfile/dss-workflow-server.Dockerfile b/k8s/dockerfile/dss-workflow-server.Dockerfile new file mode 100644 index 000000000..9983ef118 --- /dev/null +++ b/k8s/dockerfile/dss-workflow-server.Dockerfile @@ -0,0 +1,10 @@ +FROM base:1.0.0 + +WORKDIR /opt/dss + +COPY lib/dss-commons /opt/dss/dss-commons/ +COPY dss-appconns /opt/dss/dss-appconns +COPY lib/dss-orchestrator/dss-workflow-server /opt/dss/dss-workflow-server/lib/ +COPY sbin/k8s/dss-workflow-server.sh /opt/dss/dss-workflow-server/bin/startup.sh + +ENTRYPOINT ["bash","dss-workflow-server/bin/startup.sh"] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..48e341a09 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,3 @@ +{ + "lockfileVersion": 1 +} diff --git a/plugins/azkaban/linkis-jobtype/bin/config.sh b/plugins/azkaban/linkis-jobtype/bin/config.sh new file mode 100644 index 000000000..3216a8328 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/bin/config.sh @@ -0,0 +1,18 @@ +##Linkis gateway url +LINKIS_GATEWAY_URL=http://127.0.0.1:9001 + +##Linkis gateway token default WS-AUTH +LINKIS_GATEWAY_TOKEN=WS-AUTH + + +##Azkaban executor host +AZKABAN_EXECUTOR_HOST=127.0.0.1 + +### SSH Port +SSH_PORT=22 + +##Azkaban executor dir +AZKABAN_EXECUTOR_DIR=/tmp/Install/AzkabanInstall/executor + +##Azkaban executor plugin reload url +AZKABAN_EXECUTOR_URL=http://127.0.0.1:12321/executor?action=reloadJobTypePlugins \ No newline at end of file diff --git a/plugins/azkaban/linkis-jobtype/bin/install.sh b/plugins/azkaban/linkis-jobtype/bin/install.sh new file mode 100644 index 000000000..bd29452fd --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/bin/install.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +#Actively load user env +#source ~/.bash_profile + +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` + + +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 +else + echo "Succeed to" + $1 +fi +} + +##load config +echo "step1:load config" +source ${workDir}/bin/config.sh +isSuccess "load config" + +local_host="`hostname --fqdn`" + + +if test -z "$AZKABAN_EXECUTOR_HOST" +then + AZKABAN_EXECUTOR_HOST=$local_host +fi + +AZKABAN_JOBTYPE_DIR=$AZKABAN_EXECUTOR_DIR/plugins/jobtypes + +if ! ssh -p $SSH_PORT $AZKABAN_EXECUTOR_HOST test -e $AZKABAN_JOBTYPE_DIR; then + echo "ERROR:Azkaban's plugin directory does not exist!" + exit 1 +fi + +echo "start to subsitution conf" +sed -i "s#jobtype.lib.dir.*#jobtype.lib.dir=$AZKABAN_JOBTYPE_DIR/linkis/lib#g" ${workDir}/private.properties +sed -i "s#wds.linkis.gateway.url.v0.*#wds.linkis.gateway.url.v0=$LINKIS_GATEWAY_URL#g" ${workDir}/plugin.properties +sed -i "s#wds.linkis.gateway.url.v1.*#wds.linkis.gateway.url.v1=$LINKIS_GATEWAY_URL#g" ${workDir}/plugin.properties +sed -i "s#wds.linkis.client.flow.author.user.token.*#wds.linkis.client.flow.author.user.token=$LINKIS_GATEWAY_TOKEN#g" ${workDir}/plugin.properties +isSuccess "subsitution conf" + +echo "$COPY Plugin" +##ssh -p $SSH_PORT $AZKABAN_EXECUTOR_HOST "cd $AZKABAN_JOBTYPE_DIR;rm -rf linkis-bak; mv -f linkis ../linkis-bak" + +scp -P $SSH_PORT -r ${workDir} $AZKABAN_EXECUTOR_HOST:$AZKABAN_JOBTYPE_DIR + +echo "reload jobType" + +##curl $AZKABAN_EXECUTOR_URL \ No newline at end of file diff --git a/plugins/azkaban/linkis-jobtype/pom.xml b/plugins/azkaban/linkis-jobtype/pom.xml new file mode 100644 index 000000000..eaf875129 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/pom.xml @@ -0,0 +1,146 @@ + + + + + 4.0.0 + + dss + com.webank.wedatasphere.dss + 1.1.1 + + linkis-jobtype + + 0.6.1 + + + + + + org.apache.linkis + linkis-common + ${linkis.version} + + + + com.webank.wedatasphere.dss + dss-linkis-node-execution + ${dss.version} + + + linkis-common + org.apache.linkis + + + + + + com.webank.wedatasphere.schedulis + azkaban-common + ${azkaban.version} + + + com.webank.azkaban + azkaban-spi + + + com.webank.azkaban + azkaban-db + + + provided + + + + + log4j + log4j + 1.2.17 + provided + + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + 1.8 + 1.8 + UTF-8 + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + maven-assembly-plugin + 2.3 + + + jar-with-dependencies + + + + + package + + package + + + single + + + ${project.artifactId}-${project.version} + + src/main/resources/assembly.xml + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + **/*.yml + **/*.properties + **/*.sh + **/log4j2.xml + + + + + + + src/main/resources + + + ${project.artifactId}-${project.version} + + + \ No newline at end of file diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/AzkabanDssJobType.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/AzkabanDssJobType.java new file mode 100644 index 000000000..e897f4b27 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/AzkabanDssJobType.java @@ -0,0 +1,213 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype; + + +import azkaban.jobExecutor.AbstractJob; +import azkaban.utils.Props; +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.execution.impl.LinkisNodeExecutionImpl; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.job.JobTypeEnum; +import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; +import com.webank.wedatasphere.dss.linkis.node.execution.listener.LinkisExecutionListener; +import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.job.JobBuilder; +import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.log.AzkabanAppConnLog; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; + + +public class AzkabanDssJobType extends AbstractJob { + + + + private static final String SENSITIVE_JOB_PROP_NAME_SUFFIX = "_X"; + private static final String SENSITIVE_JOB_PROP_VALUE_PLACEHOLDER = "[MASKED]"; + private static final String JOB_DUMP_PROPERTIES_IN_LOG = "job.dump.properties"; + + + + private final Logger log; + + protected volatile Props jobProps; + + protected volatile Props sysProps; + + protected volatile Map jobPropsMap; + + private final String type; + + private Job job; + + private boolean isCanceled = false; + + + + + public AzkabanDssJobType(String jobId, Props sysProps, Props jobProps, Logger log) { + + + super(jobId, log); + + this.jobProps = jobProps; + + this.sysProps = sysProps; + + this.jobPropsMap = this.jobProps.getMapByPrefix(""); + + this.log = log; + this.type = jobProps.getString(JOB_TYPE, LinkisJobExecutionConfiguration.JOB_DEFAULT_TYPE.getValue(this.jobPropsMap)); + if(!LinkisJobExecutionConfiguration.JOB_DEFAULT_TYPE.getValue(this.jobPropsMap).equalsIgnoreCase(this.type) ){ + throw new RuntimeException("This job(" + this.type + " )is not linkis type"); + } + + } + + + @Override + public void run() throws Exception { + + info("Start to execute job"); + logJobProperties(); + String runDate = getRunDate(); + if (StringUtils.isNotBlank(runDate)){ + this.jobPropsMap.put("run_date", runDate); + } + this.job = JobBuilder.getAzkanbanBuilder().setJobProps(this.jobPropsMap).build(); + this.job.setLogObj(new AzkabanAppConnLog(this.log)); + if(JobTypeEnum.EmptyJob == ((LinkisJob)this.job).getJobType()){ + this.log.warn("This node is empty type"); + return; + } + // info("runtimeMap is " + job.getRuntimeParams()); + //job.getRuntimeParams().put("workspace", getWorkspace(job.getUser())); + info("runtimeMap is " + job.getRuntimeParams()); + LinkisNodeExecutionImpl.getLinkisNodeExecution().runJob(this.job); + + try { + LinkisNodeExecutionImpl.getLinkisNodeExecution().waitForComplete(this.job); + } catch (Exception e) { + this.log.warn("Failed to execute job", e); + //String reason = LinkisNodeExecutionImpl.getLinkisNodeExecution().getLog(this.job); + //this.log.error("Reason for failure: " + reason); + throw e; + } + try { + String endLog = LinkisNodeExecutionImpl.getLinkisNodeExecution().getLog(this.job); + this.log.info(endLog); + } catch (Throwable e){ + this.log.info("Failed to get log", e); + } + + LinkisExecutionListener listener = (LinkisExecutionListener)LinkisNodeExecutionImpl.getLinkisNodeExecution(); + listener.onStatusChanged(null, LinkisNodeExecutionImpl.getLinkisNodeExecution().getState(this.job),this.job); + int resultSize = 0; + try{ + resultSize = LinkisNodeExecutionImpl.getLinkisNodeExecution().getResultSize(this.job); + }catch(final Throwable t){ + this.log.error("failed to get result size"); + resultSize = -1; + } + for(int i =0; i < resultSize; i++){ + this.log.info("The content of the " + (i + 1) + "th resultset is :" + + LinkisNodeExecutionImpl.getLinkisNodeExecution().getResult(this.job, i, LinkisJobExecutionConfiguration.RESULT_PRINT_SIZE.getValue(this.jobPropsMap))); + } + + info("Finished to execute job"); + } + + @Override + public void cancel() throws Exception { + //super.cancel(); + LinkisNodeExecutionImpl.getLinkisNodeExecution().cancel(this.job); + isCanceled = true; + warn("This job has been canceled"); + } + + @Override + public boolean isCanceled() { + return isCanceled; + } + + @Override + public double getProgress() throws Exception { + return LinkisNodeExecutionImpl.getLinkisNodeExecution().getProgress(this.job); + } + + /** + * prints the current Job props to the Job log. + */ + private void logJobProperties() { + if (this.jobProps != null && + this.jobProps.getBoolean(JOB_DUMP_PROPERTIES_IN_LOG, true)) { + try { + this.info("****** Job properties ******"); + this.info(String.format("- Note : value is masked if property name ends with '%s'.", + SENSITIVE_JOB_PROP_NAME_SUFFIX)); + for (final Map.Entry entry : this.jobPropsMap.entrySet()) { + final String key = entry.getKey(); + final String value = key.endsWith(SENSITIVE_JOB_PROP_NAME_SUFFIX) ? + SENSITIVE_JOB_PROP_VALUE_PLACEHOLDER : + entry.getValue(); + this.info(String.format("%s=%s", key, value)); + } + this.info("****** End Job properties ******"); + } catch (final Exception ex) { + this.log.error("failed to log job properties ", ex); + } + } + } + + private String getRunDate(){ + this.info("begin to get run date"); + if (this.jobProps != null && + this.jobProps.getBoolean(JOB_DUMP_PROPERTIES_IN_LOG, true)) { + try { + for (final Map.Entry entry : this.jobPropsMap.entrySet()) { + final String key = entry.getKey(); + final String value = key.endsWith(SENSITIVE_JOB_PROP_NAME_SUFFIX) ? + SENSITIVE_JOB_PROP_VALUE_PLACEHOLDER : + entry.getValue(); + if ("azkaban.flow.start.timestamp".equals(key)){ + this.info("run time is " + value); + String runDateNow = value.substring(0, 10).replaceAll("-", ""); + this.info("run date now is " + runDateNow); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd"); + try { + Date date = simpleDateFormat.parse(runDateNow); + //因为date已经当天的00:00:00 减掉12小时 就是昨天的时间 + String runDate = simpleDateFormat.format(new Date(date.getTime() - 24 * 60 * 60 * 1000)); + this.info("runDate is " + runDate); + return runDate; + } catch (ParseException e) { + this.log.error("failed to parse run date " + runDateNow, e); + } + } + } + } catch (final Exception ex) { + this.log.error("failed to log job properties ", ex); + } + } + return null; + } + +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/action/WorkspaceInfoGetAction.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/action/WorkspaceInfoGetAction.java new file mode 100644 index 000000000..127438124 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/action/WorkspaceInfoGetAction.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.action; + +import org.apache.linkis.httpclient.request.GetAction; +import org.apache.linkis.httpclient.request.UserAction; + + +public class WorkspaceInfoGetAction extends GetAction implements UserAction { + + String url; + String user; + + @Override + public String getURL() { + return url; + } + + public void setURL(String url) { + this.url = url; + } + + @Override + public void setUser(String user) { + this.user = user; + } + + @Override + public String getUser() { + return user; + } + + +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/conf/LinkisJobTypeConf.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/conf/LinkisJobTypeConf.java new file mode 100644 index 000000000..d607f4512 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/conf/LinkisJobTypeConf.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.conf; + +import org.apache.linkis.common.conf.CommonVars; + + +public class LinkisJobTypeConf { + + public static final String COMMAND = "command"; + + public static final String JOB_ID = "azkaban.job.id"; + + public static final String DSS_LABELS_KEY = "labels"; + + public static final String FLOW_NAME = "azkaban.flow.flowid"; + + public static final String PROJECT_ID = "azkaban.flow.projectid"; + + public static final String PROJECT_NAME = "azkaban.flow.projectname"; + + public static final String FLOW_EXEC_ID = "azkaban.flow.execid"; + + public static final String PROXY_USER = "user.to.proxy"; + + + public static final String FLOW_SUBMIT_USER = "azkaban.flow.submituser"; + + public static final String MSG_SAVE_KEY = "msg.savekey"; + + public final static CommonVars SIGNAL_NODES = CommonVars.apply("wds.dss.flow.signal.nodes","linkis.appconn.eventchecker.eventreceiver"); + +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanAppConnLinkisJob.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanAppConnLinkisJob.java new file mode 100644 index 000000000..3de788c44 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanAppConnLinkisJob.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.job.AbstractAppConnLinkisJob; +import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.conf.LinkisJobTypeConf; + + +public class AzkabanAppConnLinkisJob extends AbstractAppConnLinkisJob { + + + @Override + public String getSubmitUser() { + return getJobProps().get(LinkisJobTypeConf.FLOW_SUBMIT_USER); + } + + + @Override + public String getUser() { + return getJobProps().get(LinkisJobTypeConf.PROXY_USER); + } + + @Override + public String getJobName() { + return getJobProps().get(LinkisJobTypeConf.JOB_ID); + } + + +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanCommonLinkisJob.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanCommonLinkisJob.java new file mode 100644 index 000000000..eed1f8109 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanCommonLinkisJob.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.job.AbstractCommonLinkisJob; +import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.conf.LinkisJobTypeConf; + + +public class AzkabanCommonLinkisJob extends AbstractCommonLinkisJob { + + + @Override + public String getSubmitUser() { + return getJobProps().get(LinkisJobTypeConf.FLOW_SUBMIT_USER); + } + + + @Override + public String getUser() { + return getJobProps().get(LinkisJobTypeConf.PROXY_USER); + } + + @Override + public String getJobName() { + return getJobProps().get(LinkisJobTypeConf.JOB_ID); + } + + +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java new file mode 100644 index 000000000..b6af36d63 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java @@ -0,0 +1,202 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Builder; +import com.webank.wedatasphere.dss.linkis.node.execution.job.CommonLinkisJob; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; +import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.conf.LinkisJobTypeConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + + +public class AzkanbanBuilder extends Builder { + + private static final Logger LOGGER = LoggerFactory.getLogger(AzkanbanBuilder.class); + + private static final String RUN_DATE_KEY = "run_date"; + private static final String RUN_DATE_HOUR_KEY = "run_date_h"; + private Map jobProps; + + public AzkanbanBuilder setJobProps(Map jobProps) { + this.jobProps = jobProps; + return this; + } + + @Override + protected String getJobType() { + return jobProps.getOrDefault(LinkisJobExecutionConfiguration.LINKIS_TYPE, + LinkisJobExecutionConfiguration.LINKIS_DEFAULT_TYPE.getValue(jobProps)); + } + + @Override + protected LinkisJob creatLinkisJob(boolean isLinkisType) { + if (isLinkisType) { + AzkabanCommonLinkisJob linkisJob = new AzkabanCommonLinkisJob(); + linkisJob.setJobProps(this.jobProps); + return linkisJob; + } else { + AzkabanAppConnLinkisJob linkisJob = new AzkabanAppConnLinkisJob(); + linkisJob.setJobProps(this.jobProps); + return linkisJob; + } + } + + @Override + protected void fillJobInfo(Job job) { + job.setCode(jobProps.get(LinkisJobTypeConf.COMMAND)); + + Map params = new HashMap<>(); + if (jobProps.containsKey("run_date")) { + params.put("run_date", jobProps.get("run_date")); + } + job.setParams(params); + + Map runtimeMap = new HashMap<>(); + if (null != job.getRuntimeParams()) { + runtimeMap = job.getRuntimeParams(); + } + + runtimeMap.put("nodeName", jobProps.get(LinkisJobTypeConf.JOB_ID)); + + runtimeMap.put(LinkisJobTypeConf.DSS_LABELS_KEY, jobProps.get(LinkisJobTypeConf.DSS_LABELS_KEY)); + job.setRuntimeParams(runtimeMap); + } + + @Override + protected String getContextID(Job job) { + return jobProps.get(LinkisJobExecutionConfiguration.FLOW_CONTEXTID); + } + + @Override + protected void fillLinkisJobInfo(LinkisJob linkisJob) { + linkisJob.setConfiguration(findConfiguration(LinkisJobExecutionConfiguration.NODE_CONF_PREFIX)); + Map variables = findVariables(LinkisJobExecutionConfiguration.FLOW_VARIABLE_PREFIX); + // 只有工作流参数中没有设置,我们才会去进行替换 + // 改为不管工作流是否设置,在 Schedulis 这边都需要统一使用 Schedulis 设置的 run_date和un_date_h,防止出现批量调度的误导作用 + setNewRunDateVariable(variables, RUN_DATE_KEY); + setNewRunDateVariable(variables, RUN_DATE_HOUR_KEY); + linkisJob.setVariables(variables); + linkisJob.setSource(getSource()); + } + + private void setNewRunDateVariable(Map variables, String replaceVar) { + if (jobProps.containsKey(replaceVar)) { + variables.put(replaceVar, jobProps.get(replaceVar)); + LOGGER.info("Put {} to variables,value: {}", replaceVar, jobProps.get(replaceVar)); + } + } + + @Override + protected void fillCommonLinkisJobInfo(CommonLinkisJob linkisAppConnJob) { + linkisAppConnJob.setJobResourceList(LinkisJobExecutionUtils.getResourceListByJson(jobProps.get("resources"))); + + String projectResourceName = LinkisJobExecutionConfiguration.PROJECT_PREFIX + "." + + jobProps.get(LinkisJobTypeConf.PROJECT_NAME) + LinkisJobExecutionConfiguration.RESOURCES_NAME; + linkisAppConnJob.setProjectResourceList(LinkisJobExecutionUtils.getResourceListByJson(jobProps.get(projectResourceName))); + + linkisAppConnJob.setFlowNameAndResources(findFLowNameAndResources()); + } + + + private Map getSource() { + Map source = new HashMap<>(); + source.put("projectName", jobProps.get(LinkisJobTypeConf.PROJECT_NAME)); + source.put("flowName", jobProps.get(LinkisJobTypeConf.FLOW_NAME)); + source.put("nodeName", jobProps.get(LinkisJobTypeConf.JOB_ID)); + return source; + } + + /** + * Looking for custom variables through the corresponding prefix + * For example, flow.variable.a=test returns map{a->test} + * + * @param prefix + * @return + */ + private Map findVariables(String prefix) { + Map map = new HashMap<>(); + Iterator keyIterator = jobProps.keySet().iterator(); + while (keyIterator.hasNext()) { + String next = keyIterator.next(); + if (next.startsWith(prefix)) { + map.put(next.substring(prefix.length()), jobProps.get(next)); + } + } + return map; + } + + /** + * Looking for configuration through the corresponding prefix includes startup,runtime,special etc + * + * @param prefix + * @return + */ + private Map findConfiguration(String prefix) { + Map configuration = new HashMap<>(); + Iterator keyIterator = jobProps.keySet().iterator(); + while (keyIterator.hasNext()) { + String next = keyIterator.next(); + if (next.startsWith(prefix)) { + String confTypeAndName = next.substring(prefix.length()); + if (confTypeAndName.startsWith(LinkisJobExecutionConfiguration.CONF_STARTUP)) { + putConf(configuration, LinkisJobExecutionConfiguration.CONF_STARTUP, jobProps.get(next), confTypeAndName); + } else if (confTypeAndName.startsWith(LinkisJobExecutionConfiguration.CONF_RUNTIME)) { + putConf(configuration, LinkisJobExecutionConfiguration.CONF_RUNTIME, jobProps.get(next), confTypeAndName); + } else if (confTypeAndName.startsWith(LinkisJobExecutionConfiguration.CONF_SPECIAL)) { + putConf(configuration, LinkisJobExecutionConfiguration.CONF_SPECIAL, jobProps.get(next), confTypeAndName); + } + } + } + return configuration; + } + + private void putConf(Map configuration, String key, String value, String confTypeAndName) { + if (configuration.get(key) == null) { + Map startup = new HashMap<>(); + startup.put(confTypeAndName.substring(key.length() + 1), value); + configuration.put(key, startup); + } else { + Map startup = (Map) configuration.get(key); + startup.put(confTypeAndName.substring(key.length() + 1), value); + } + } + + + private Map> findFLowNameAndResources() { + Map> flowNameAndResources = new HashMap<>(); + Iterator iterator = jobProps.keySet().iterator(); + while (iterator.hasNext()) { + String next = iterator.next(); + if (next.endsWith("resources") && next.startsWith("flow")) { + flowNameAndResources.put(next, LinkisJobExecutionUtils.getResourceListByJson(jobProps.get(next))); + } + } + return flowNameAndResources; + } + + +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/JobBuilder.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/JobBuilder.java new file mode 100644 index 000000000..a206f1781 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/JobBuilder.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.job; + + +public class JobBuilder { + + public static AzkanbanBuilder getAzkanbanBuilder(){ + return new AzkanbanBuilder(); + } + +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/log/AzkabanAppConnLog.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/log/AzkabanAppConnLog.java new file mode 100644 index 000000000..70f3c7315 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/log/AzkabanAppConnLog.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.log; + +import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; +import org.slf4j.Logger; + + +public class AzkabanAppConnLog extends LinkisJobExecutionLog { + + private Logger log; + + public AzkabanAppConnLog(Logger log){ + this.log = log; + } + + + + @Override + public void info(Object message, Throwable t) { + log.info(message.toString(), t); + } + + @Override + public void warn(Object message, Throwable t) { + log.warn(message.toString(), t); + } + + @Override + public void error(Object message, Throwable t) { + log.error(message.toString(), t); + } +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/resources/assembly.xml b/plugins/azkaban/linkis-jobtype/src/main/resources/assembly.xml new file mode 100644 index 000000000..e2342df7d --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/resources/assembly.xml @@ -0,0 +1,83 @@ + + + + linkis-jobtype + + zip + + true + linkis + + + + lib + true + true + false + true + true + + + + + + + + + ${basedir}/src/main/resources + + plugin.properties + + 0777 + / + unix + + + + ${basedir}/src/main/resources + + private.properties + + 0777 + / + unix + + + + ${basedir}/target + + *.jar + + 0777 + / + unix + + + ${basedir}/bin + + install.sh + config.sh + + 0777 + bin + unix + + + + diff --git a/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties b/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties new file mode 100644 index 000000000..e5d15ce7b --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties @@ -0,0 +1,21 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +wds.linkis.gateway.url.v1=http://127.0.0.1:9001 +wds.linkis.gateway.url.v0=http://127.0.0.1:9001 +wds.linkis.client.flow.author.user.token=WS-AUTH +wds.linkis.flow.job.creator=scheduler +wds.linkis.flow.job.creator.v1=schedulis diff --git a/plugins/azkaban/linkis-jobtype/src/main/resources/private.properties b/plugins/azkaban/linkis-jobtype/src/main/resources/private.properties new file mode 100644 index 000000000..c3d0200d8 --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/resources/private.properties @@ -0,0 +1,18 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +jobtype.class=com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.AzkabanDssJobType +jobtype.lib.dir=/tmp/AzkabanInstall/executor/plugins/jobtypes/linkis/lib diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml new file mode 100644 index 000000000..fdad1eadd --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml @@ -0,0 +1,112 @@ + + + + + + dss-plugin-dolphinscheduler + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dolphinscheduler-prod-metrics + + + + org.apache.dolphinscheduler + dolphinscheduler-api + ${dolphinscheduler.version} + provided + + + com.webank.wedatasphere.dss + spring-origin-sso-integration-plugin + ${dss.version} + + + mysql + mysql-connector-java + + + org.springframework + spring-core + + + spring-webmvc + org.springframework + + + spring-context-support + org.springframework + + + spring-aop + org.springframework + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + + + jar-with-dependencies + + + + + assembly + package + + single + + + + + + + + src/main/resources + + **/*.xml + + + + + + \ No newline at end of file diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/configuration/DssSSOConfiguration.java b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/configuration/DssSSOConfiguration.java new file mode 100644 index 000000000..14df79643 --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/configuration/DssSSOConfiguration.java @@ -0,0 +1,25 @@ +package org.apache.dolphinscheduler.api.configuration; + +import com.webank.wedatasphere.dss.standard.app.sso.origin.filter.spring.SpringOriginSSOPluginFilter; +import com.webank.wedatasphere.dss.standard.app.sso.plugin.filter.SSOPluginFilter; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author enjoyyin + * @date 2022-11-16 + * @since 1.1.1 + */ +@Configuration +public class DssSSOConfiguration { + + @Bean + public FilterRegistrationBean dssSSOInterceptor() { + FilterRegistrationBean filter = new FilterRegistrationBean<>(); + filter.setName("dssSSOFilter"); + filter.setFilter(new SpringOriginSSOPluginFilter()); + filter.setOrder(-1); + return filter; + } +} diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceMetricsController.java b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceMetricsController.java new file mode 100644 index 000000000..985b317d9 --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceMetricsController.java @@ -0,0 +1,95 @@ +package org.apache.dolphinscheduler.api.controller; + +import io.swagger.annotations.*; +import org.apache.dolphinscheduler.api.exceptions.ApiException; +import org.apache.dolphinscheduler.api.service.ProcessInstanceMetricsService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.dao.entity.User; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; + +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_LIST_ORDER_BY_DURATION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_STATISTICS_ERROR; + +import java.util.List; +import java.util.Map; + +/** + * @author enjoyyin + * @date 2022-02-23 + * @since 0.5.0 + */ +@Api(tags = "PROCESS_INSTANCE_METRICS_TAG") +@RestController +@RequestMapping("/projects/{projectName}/instance") +public class ProcessInstanceMetricsController extends BaseController { + + private static final Logger logger = LoggerFactory.getLogger(ProcessInstanceMetricsController.class); + + @Autowired + private ProcessInstanceMetricsService processInstanceService; + + { + logger.info("dolphinscheduler-prod-metrics module is loaded."); + } + + @ApiOperation(value = "queryProcessInstanceListOrderByDuration", + notes = "QUERY_PROCESS_INSTANCE_LIST_ORDER_BY_DURATION_NOTES") + @ApiImplicitParams( + {@ApiImplicitParam(name = "processDefinitionId", value = "PROCESS_DEFINITION_ID", dataType = "Int", + example = "100"), @ApiImplicitParam(name = "executorName", value = "EXECUTOR_NAME", type = "String"), + @ApiImplicitParam(name = "startDate", value = "START_DATE", type = "String"), + @ApiImplicitParam(name = "endDate", value = "END_DATE", type = "String"), + @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", dataType = "Int", example = "1"), + @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", dataType = "Int", example = "10")}) + @GetMapping(value = "list-order-by-duration") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_PROCESS_INSTANCE_LIST_ORDER_BY_DURATION_ERROR) + public Result queryProcessInstanceListOrderByDuration( + @ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "processDefinitionId", required = false, defaultValue = "0") Integer processDefinitionId, + @RequestParam(value = "executorName", required = false) String executorName, + @RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, @RequestParam("pageNo") Integer pageNo, + @RequestParam("pageSize") Integer pageSize) { + logger.info("query process instance list order by duration, login user:{}, project name:{}, define id:{}, executor name:{}, start time:{}, end time:{}, page number:{}, page size:{}", + loginUser.getUserName(), projectName, processDefinitionId, executorName, startDate, endDate, pageNo, + pageSize); + Map result = processInstanceService.queryProcessInstanceListOrderByDuration(loginUser, + projectName, processDefinitionId, executorName, startDate, endDate, pageNo, pageSize); + return returnDataListPaging(result); + } + + @ApiOperation(value = "queryProcessInstanceStatistics", notes = "QUERY_PROCESS_INSTANCE_STATISTICS_NOTES") + @ApiImplicitParams( + {@ApiImplicitParam(name = "processDefinitionId", value = "PROCESS_DEFINITION_ID", dataType = "Int", + defaultValue = "0"), @ApiImplicitParam(name = "executorName", value = "EXECUTOR_NAME", type = "String"), + @ApiImplicitParam(name = "dates", value = "DATES", type = "String"), + @ApiImplicitParam(name = "stateType", value = "EXECUTION_STATUS", type = "ExecutionStatus"), + @ApiImplicitParam(name = "step", value = "STEP", type = "Int")}) + @GetMapping(value = "statistics") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_PROCESS_INSTANCE_STATISTICS_ERROR) + public Result queryProcessInstanceStatistics( + @ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "processDefinitionId", required = false, defaultValue = "0") Integer processDefinitionId, + @RequestParam(value = "executorName", required = false) String executorName, + @RequestParam(value = "dates") List dates, @RequestParam(value = "stateType") ExecutionStatus stateType, + @RequestParam(value = "step") int step) { + logger.info( + "query process instance statistics, login user:{}, project name:{}, define id:{}, executor name:{}, dates:{}, state:{}", + loginUser.getUserName(), projectName, processDefinitionId, executorName, dates, stateType); + Map result = processInstanceService.queryProcessInstanceStatistics(loginUser, projectName, + processDefinitionId, executorName, dates, stateType.ordinal(), step); + return returnDataList(result); + } + +} diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/enums/MetricsStatus.java b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/enums/MetricsStatus.java new file mode 100644 index 000000000..ce0980615 --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/enums/MetricsStatus.java @@ -0,0 +1,40 @@ +package org.apache.dolphinscheduler.api.enums; + +import org.springframework.context.i18n.LocaleContextHolder; + +import java.util.Locale; + +/** + * @author enjoyyin + * @date 2022-02-23 + * @since 0.5.0 + */ +public enum MetricsStatus { + + QUERY_PROCESS_INSTANCE_LIST_ORDER_BY_DURATION_ERROR(10113, "query process instance list order by duration error", + "分页查询工作流实例列表(耗时降序)错误"), + QUERY_PROCESS_INSTANCE_STATISTICS_ERROR(10113, "query process instance statistics error", "查询周期内实例完成情况错误"); + + private final int code; + private final String enMsg; + private final String zhMsg; + + private MetricsStatus(int code, String enMsg, String zhMsg) { + this.code = code; + this.enMsg = enMsg; + this.zhMsg = zhMsg; + } + + public int getCode() { + return this.code; + } + + public String getMsg() { + if (Locale.SIMPLIFIED_CHINESE.getLanguage().equals(LocaleContextHolder.getLocale().getLanguage())) { + return this.zhMsg; + } else { + return this.enMsg; + } + } + +} diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java new file mode 100644 index 000000000..b892eacd7 --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -0,0 +1,277 @@ +package org.apache.dolphinscheduler.api.enums; + +import org.springframework.context.i18n.LocaleContextHolder; + +import java.util.Locale; + +/** + * @author enjoyyin + * @date 2022-02-23 + * @since 0.5.0 + */ +public enum Status { + SUCCESS(0, "success", "成功"), + + INTERNAL_SERVER_ERROR_ARGS(10000, "Internal Server Error: {0}", "服务端异常: {0}"), + + REQUEST_PARAMS_NOT_VALID_ERROR(10001, "request parameter {0} is not valid", "请求参数[{0}]无效"), + TASK_TIMEOUT_PARAMS_ERROR(10002, "task timeout parameter is not valid", "任务超时参数无效"), + USER_NAME_EXIST(10003, "user name already exists", "用户名已存在"), + USER_NAME_NULL(10004,"user name is null", "用户名不能为空"), + HDFS_OPERATION_ERROR(10006, "hdfs operation error", "hdfs操作错误"), + TASK_INSTANCE_NOT_FOUND(10008, "task instance not found", "任务实例不存在"), + TENANT_NAME_EXIST(10009, "tenant code {0} already exists", "租户编码[{0}]已存在"), + USER_NOT_EXIST(10010, "user {0} not exists", "用户[{0}]不存在"), + ALERT_GROUP_NOT_EXIST(10011, "alarm group not found", "告警组不存在"), + ALERT_GROUP_EXIST(10012, "alarm group already exists", "告警组名称已存在"), + USER_NAME_PASSWD_ERROR(10013,"user name or password error", "用户名或密码错误"), + LOGIN_SESSION_FAILED(10014,"create session failed!", "创建session失败"), + DATASOURCE_EXIST(10015, "data source name already exists", "数据源名称已存在"), + DATASOURCE_CONNECT_FAILED(10016, "data source connection failed", "建立数据源连接失败"), + TENANT_NOT_EXIST(10017, "tenant not exists", "租户不存在"), + PROJECT_NOT_FOUNT(10018, "project {0} not found ", "项目[{0}]不存在"), + PROJECT_ALREADY_EXISTS(10019, "project {0} already exists", "项目名称[{0}]已存在"), + TASK_INSTANCE_NOT_EXISTS(10020, "task instance {0} does not exist", "任务实例[{0}]不存在"), + TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE(10021, "task instance {0} is not sub process instance", "任务实例[{0}]不是子流程实例"), + SCHEDULE_CRON_NOT_EXISTS(10022, "scheduler crontab {0} does not exist", "调度配置定时表达式[{0}]不存在"), + SCHEDULE_CRON_ONLINE_FORBID_UPDATE(10023, "online status does not allow update operations", "调度配置上线状态不允许修改"), + SCHEDULE_CRON_CHECK_FAILED(10024, "scheduler crontab expression validation failure: {0}", "调度配置定时表达式验证失败: {0}"), + MASTER_NOT_EXISTS(10025, "master does not exist", "无可用master节点"), + SCHEDULE_STATUS_UNKNOWN(10026, "unknown status: {0}", "未知状态: {0}"), + CREATE_ALERT_GROUP_ERROR(10027,"create alert group error", "创建告警组错误"), + QUERY_ALL_ALERTGROUP_ERROR(10028,"query all alertgroup error", "查询告警组错误"), + LIST_PAGING_ALERT_GROUP_ERROR(10029,"list paging alert group error", "分页查询告警组错误"), + UPDATE_ALERT_GROUP_ERROR(10030,"update alert group error", "更新告警组错误"), + DELETE_ALERT_GROUP_ERROR(10031,"delete alert group error", "删除告警组错误"), + ALERT_GROUP_GRANT_USER_ERROR(10032,"alert group grant user error", "告警组授权用户错误"), + CREATE_DATASOURCE_ERROR(10033,"create datasource error", "创建数据源错误"), + UPDATE_DATASOURCE_ERROR(10034,"update datasource error", "更新数据源错误"), + QUERY_DATASOURCE_ERROR(10035,"query datasource error", "查询数据源错误"), + CONNECT_DATASOURCE_FAILURE(10036,"connect datasource failure", "建立数据源连接失败"), + CONNECTION_TEST_FAILURE(10037,"connection test failure", "测试数据源连接失败"), + DELETE_DATA_SOURCE_FAILURE(10038,"delete data source failure", "删除数据源失败"), + VERIFY_DATASOURCE_NAME_FAILURE(10039,"verify datasource name failure", "验证数据源名称失败"), + UNAUTHORIZED_DATASOURCE(10040,"unauthorized datasource", "未经授权的数据源"), + AUTHORIZED_DATA_SOURCE(10041,"authorized data source", "授权数据源失败"), + LOGIN_SUCCESS(10042,"login success", "登录成功"), + USER_LOGIN_FAILURE(10043,"user login failure", "用户登录失败"), + LIST_WORKERS_ERROR(10044,"list workers error", "查询worker列表错误"), + LIST_MASTERS_ERROR(10045,"list masters error", "查询master列表错误"), + UPDATE_PROJECT_ERROR(10046,"update project error", "更新项目信息错误"), + QUERY_PROJECT_DETAILS_BY_ID_ERROR(10047,"query project details by id error", "查询项目详细信息错误"), + CREATE_PROJECT_ERROR(10048,"create project error", "创建项目错误"), + LOGIN_USER_QUERY_PROJECT_LIST_PAGING_ERROR(10049,"login user query project list paging error", "分页查询项目列表错误"), + DELETE_PROJECT_ERROR(10050,"delete project error", "删除项目错误"), + QUERY_UNAUTHORIZED_PROJECT_ERROR(10051,"query unauthorized project error", "查询未授权项目错误"), + QUERY_AUTHORIZED_PROJECT(10052,"query authorized project", "查询授权项目错误"), + QUERY_QUEUE_LIST_ERROR(10053,"query queue list error", "查询队列列表错误"), + CREATE_RESOURCE_ERROR(10054,"create resource error", "创建资源错误"), + UPDATE_RESOURCE_ERROR(10055,"update resource error", "更新资源错误"), + QUERY_RESOURCES_LIST_ERROR(10056,"query resources list error", "查询资源列表错误"), + QUERY_RESOURCES_LIST_PAGING(10057,"query resources list paging", "分页查询资源列表错误"), + DELETE_RESOURCE_ERROR(10058,"delete resource error", "删除资源错误"), + VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR(10059,"verify resource by name and type error", "资源名称或类型验证错误"), + VIEW_RESOURCE_FILE_ON_LINE_ERROR(10060,"view resource file online error", "查看资源文件错误"), + CREATE_RESOURCE_FILE_ON_LINE_ERROR(10061,"create resource file online error", "创建资源文件错误"), + RESOURCE_FILE_IS_EMPTY(10062,"resource file is empty", "资源文件内容不能为空"), + EDIT_RESOURCE_FILE_ON_LINE_ERROR(10063,"edit resource file online error", "更新资源文件错误"), + DOWNLOAD_RESOURCE_FILE_ERROR(10064,"download resource file error", "下载资源文件错误"), + CREATE_UDF_FUNCTION_ERROR(10065 ,"create udf function error", "创建UDF函数错误"), + VIEW_UDF_FUNCTION_ERROR( 10066,"view udf function error", "查询UDF函数错误"), + UPDATE_UDF_FUNCTION_ERROR(10067,"update udf function error", "更新UDF函数错误"), + QUERY_UDF_FUNCTION_LIST_PAGING_ERROR( 10068,"query udf function list paging error", "分页查询UDF函数列表错误"), + QUERY_DATASOURCE_BY_TYPE_ERROR( 10069,"query datasource by type error", "查询数据源信息错误"), + VERIFY_UDF_FUNCTION_NAME_ERROR( 10070,"verify udf function name error", "UDF函数名称验证错误"), + DELETE_UDF_FUNCTION_ERROR( 10071,"delete udf function error", "删除UDF函数错误"), + AUTHORIZED_FILE_RESOURCE_ERROR( 10072,"authorized file resource error", "授权资源文件错误"), + AUTHORIZE_RESOURCE_TREE( 10073,"authorize resource tree display error","授权资源目录树错误"), + UNAUTHORIZED_UDF_FUNCTION_ERROR( 10074,"unauthorized udf function error", "查询未授权UDF函数错误"), + AUTHORIZED_UDF_FUNCTION_ERROR(10075,"authorized udf function error", "授权UDF函数错误"), + CREATE_SCHEDULE_ERROR(10076,"create schedule error", "创建调度配置错误"), + UPDATE_SCHEDULE_ERROR(10077,"update schedule error", "更新调度配置错误"), + PUBLISH_SCHEDULE_ONLINE_ERROR(10078,"publish schedule online error", "上线调度配置错误"), + OFFLINE_SCHEDULE_ERROR(10079,"offline schedule error", "下线调度配置错误"), + QUERY_SCHEDULE_LIST_PAGING_ERROR(10080,"query schedule list paging error", "分页查询调度配置列表错误"), + QUERY_SCHEDULE_LIST_ERROR(10081,"query schedule list error", "查询调度配置列表错误"), + QUERY_TASK_LIST_PAGING_ERROR(10082,"query task list paging error", "分页查询任务列表错误"), + QUERY_TASK_RECORD_LIST_PAGING_ERROR(10083,"query task record list paging error", "分页查询任务记录错误"), + CREATE_TENANT_ERROR(10084,"create tenant error", "创建租户错误"), + QUERY_TENANT_LIST_PAGING_ERROR(10085,"query tenant list paging error", "分页查询租户列表错误"), + QUERY_TENANT_LIST_ERROR(10086,"query tenant list error", "查询租户列表错误"), + UPDATE_TENANT_ERROR(10087,"update tenant error", "更新租户错误"), + DELETE_TENANT_BY_ID_ERROR(10088,"delete tenant by id error", "删除租户错误"), + VERIFY_TENANT_CODE_ERROR(10089,"verify tenant code error", "租户编码验证错误"), + CREATE_USER_ERROR(10090,"create user error", "创建用户错误"), + QUERY_USER_LIST_PAGING_ERROR(10091,"query user list paging error", "分页查询用户列表错误"), + UPDATE_USER_ERROR(10092,"update user error", "更新用户错误"), + DELETE_USER_BY_ID_ERROR(10093,"delete user by id error", "删除用户错误"), + GRANT_PROJECT_ERROR(10094,"grant project error", "授权项目错误"), + GRANT_RESOURCE_ERROR(10095,"grant resource error", "授权资源错误"), + GRANT_UDF_FUNCTION_ERROR(10096,"grant udf function error", "授权UDF函数错误"), + GRANT_DATASOURCE_ERROR(10097,"grant datasource error", "授权数据源错误"), + GET_USER_INFO_ERROR(10098,"get user info error", "获取用户信息错误"), + USER_LIST_ERROR(10099,"user list error", "查询用户列表错误"), + VERIFY_USERNAME_ERROR(10100,"verify username error", "用户名验证错误"), + UNAUTHORIZED_USER_ERROR(10101,"unauthorized user error", "查询未授权用户错误"), + AUTHORIZED_USER_ERROR(10102,"authorized user error", "查询授权用户错误"), + QUERY_TASK_INSTANCE_LOG_ERROR(10103,"view task instance log error", "查询任务实例日志错误"), + DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104,"download task instance log file error", "下载任务日志文件错误"), + CREATE_PROCESS_DEFINITION(10105,"create process definition", "创建工作流错误"), + VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106,"verify process definition name unique error", "工作流定义名称已存在"), + UPDATE_PROCESS_DEFINITION_ERROR(10107,"update process definition error", "更新工作流定义错误"), + RELEASE_PROCESS_DEFINITION_ERROR(10108,"release process definition error", "上线工作流错误"), + QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109,"query datail of process definition error", "查询工作流详细信息错误"), + QUERY_PROCESS_DEFINITION_LIST(10110,"query process definition list", "查询工作流列表错误"), + ENCAPSULATION_TREEVIEW_STRUCTURE_ERROR(10111,"encapsulation treeview structure error", "查询工作流树形图数据错误"), + GET_TASKS_LIST_BY_PROCESS_DEFINITION_ID_ERROR(10112,"get tasks list by process definition id error", "查询工作流定义节点信息错误"), + QUERY_PROCESS_INSTANCE_LIST_PAGING_ERROR(10113,"query process instance list paging error", "分页查询工作流实例列表错误"), + QUERY_TASK_LIST_BY_PROCESS_INSTANCE_ID_ERROR(10114,"query task list by process instance id error", "查询任务实例列表错误"), + UPDATE_PROCESS_INSTANCE_ERROR(10115,"update process instance error", "更新工作流实例错误"), + QUERY_PROCESS_INSTANCE_BY_ID_ERROR(10116,"query process instance by id error", "查询工作流实例错误"), + DELETE_PROCESS_INSTANCE_BY_ID_ERROR(10117,"delete process instance by id error", "删除工作流实例错误"), + QUERY_SUB_PROCESS_INSTANCE_DETAIL_INFO_BY_TASK_ID_ERROR(10118,"query sub process instance detail info by task id error", "查询子流程任务实例错误"), + QUERY_PARENT_PROCESS_INSTANCE_DETAIL_INFO_BY_SUB_PROCESS_INSTANCE_ID_ERROR(10119,"query parent process instance detail info by sub process instance id error", "查询子流程该工作流实例错误"), + QUERY_PROCESS_INSTANCE_ALL_VARIABLES_ERROR(10120,"query process instance all variables error", "查询工作流自定义变量信息错误"), + ENCAPSULATION_PROCESS_INSTANCE_GANTT_STRUCTURE_ERROR(10121,"encapsulation process instance gantt structure error", "查询工作流实例甘特图数据错误"), + QUERY_PROCESS_DEFINITION_LIST_PAGING_ERROR(10122,"query process definition list paging error", "分页查询工作流定义列表错误"), + SIGN_OUT_ERROR(10123,"sign out error", "退出错误"), + TENANT_CODE_HAS_ALREADY_EXISTS(10124,"tenant code has already exists", "租户编码已存在"), + IP_IS_EMPTY(10125,"ip is empty", "IP地址不能为空"), + SCHEDULE_CRON_REALEASE_NEED_NOT_CHANGE(10126, "schedule release is already {0}", "调度配置上线错误[{0}]"), + CREATE_QUEUE_ERROR(10127, "create queue error", "创建队列错误"), + QUEUE_NOT_EXIST(10128, "queue {0} not exists", "队列ID[{0}]不存在"), + QUEUE_VALUE_EXIST(10129, "queue value {0} already exists", "队列值[{0}]已存在"), + QUEUE_NAME_EXIST(10130, "queue name {0} already exists", "队列名称[{0}]已存在"), + UPDATE_QUEUE_ERROR(10131, "update queue error", "更新队列信息错误"), + NEED_NOT_UPDATE_QUEUE(10132, "no content changes, no updates are required", "数据未变更,不需要更新队列信息"), + VERIFY_QUEUE_ERROR(10133,"verify queue error", "验证队列信息错误"), + NAME_NULL(10134,"name must be not null", "名称不能为空"), + NAME_EXIST(10135, "name {0} already exists", "名称[{0}]已存在"), + SAVE_ERROR(10136, "save error", "保存错误"), + DELETE_PROJECT_ERROR_DEFINES_NOT_NULL(10137, "please delete the process definitions in project first!", "请先删除全部工作流定义"), + BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117,"batch delete process instance by ids {0} error", "批量删除工作流实例错误"), + PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error", "预览调度配置错误"), + PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error", "解析调度表达式错误"), + SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end", "开始时间不能和结束时间一样"), + DELETE_TENANT_BY_ID_FAIL(10142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败,有[{0}]个运行中的工作流实例正在使用"), + DELETE_TENANT_BY_ID_FAIL_DEFINES(10143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败,有[{0}]个工作流定义正在使用"), + DELETE_TENANT_BY_ID_FAIL_USERS(10144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败,有[{0}]个用户正在使用"), + DELETE_WORKER_GROUP_BY_ID_FAIL(10145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败,有[{0}]个运行中的工作流实例正在使用"), + QUERY_WORKER_GROUP_FAIL(10146,"query worker group fail ", "查询worker分组失败"), + DELETE_WORKER_GROUP_FAIL(10147,"delete worker group fail ", "删除worker分组失败"), + COPY_PROCESS_DEFINITION_ERROR(10148,"copy process definition error", "复制工作流错误"), + DATASOURCE_DB_TYPE_ILLEGAL(10169, "datasource type illegal", "数据源类型参数不合法"), + DATASOURCE_PORT_ILLEGAL(10170, "datasource port illegal", "数据源端口参数不合法"), + DATASOURCE_OTHER_PARAMS_ILLEGAL(10171, "datasource other params illegal", "数据源其他参数不合法"), + DATASOURCE_NAME_ILLEGAL(10172, "datasource name illegal", "数据源名称不合法"), + DATASOURCE_HOST_ILLEGAL(10173, "datasource host illegal", "数据源HOST不合法"), + DELETE_WORKER_GROUP_NOT_EXIST(10174, "delete worker group not exist ", "删除worker分组不存在"), + CREATE_WORKER_GROUP_FORBIDDEN_IN_DOCKER(10175, "create worker group forbidden in docker ", "创建worker分组在docker中禁止"), + DELETE_WORKER_GROUP_FORBIDDEN_IN_DOCKER(10176, "delete worker group forbidden in docker ", "删除worker分组在docker中禁止"), + WORKER_ADDRESS_INVALID(10177, "worker address {0} invalid", "worker地址[{0}]无效"), + QUERY_WORKER_ADDRESS_LIST_FAIL(10178, "query worker address list fail ", "查询worker地址列表失败"), + + UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), + UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), + RESOURCE_NOT_EXIST(20004, "resource not exist", "资源不存在"), + RESOURCE_EXIST(20005, "resource already exists", "资源已存在"), + RESOURCE_SUFFIX_NOT_SUPPORT_VIEW(20006, "resource suffix do not support online viewing", "资源文件后缀不支持查看"), + RESOURCE_SIZE_EXCEED_LIMIT(20007, "upload resource file size exceeds limit", "上传资源文件大小超过限制"), + RESOURCE_SUFFIX_FORBID_CHANGE(20008, "resource suffix not allowed to be modified", "资源文件后缀不支持修改"), + UDF_RESOURCE_SUFFIX_NOT_JAR(20009, "UDF resource suffix name must be jar", "UDF资源文件后缀名只支持[jar]"), + HDFS_COPY_FAIL(20010, "hdfs copy {0} -> {1} fail", "hdfs复制失败:[{0}] -> [{1}]"), + RESOURCE_FILE_EXIST(20011, "resource file {0} already exists in hdfs,please delete it or change name!", "资源文件[{0}]在hdfs中已存在,请删除或修改资源名"), + RESOURCE_FILE_NOT_EXIST(20012, "resource file {0} not exists in hdfs!", "资源文件[{0}]在hdfs中不存在"), + UDF_RESOURCE_IS_BOUND(20013, "udf resource file is bound by UDF functions:{0}","udf函数绑定了资源文件[{0}]"), + RESOURCE_IS_USED(20014, "resource file is used by process definition","资源文件被上线的流程定义使用了"), + PARENT_RESOURCE_NOT_EXIST(20015, "parent resource not exist","父资源文件不存在"), + RESOURCE_NOT_EXIST_OR_NO_PERMISSION(20016, "resource not exist or no permission,please view the task node and remove error resource","请检查任务节点并移除无权限或者已删除的资源"), + RESOURCE_IS_AUTHORIZED(20017, "resource is authorized to user {0},suffix not allowed to be modified", "资源文件已授权其他用户[{0}],后缀不允许修改"), + + USER_NO_OPERATION_PERM(30001, "user has no operation privilege", "当前用户没有操作权限"), + USER_NO_OPERATION_PROJECT_PERM(30002, "user {0} is not has project {1} permission", "当前用户[{0}]没有[{1}]项目的操作权限"), + + + PROCESS_INSTANCE_NOT_EXIST(50001, "process instance {0} does not exist", "工作流实例[{0}]不存在"), + PROCESS_INSTANCE_EXIST(50002, "process instance {0} already exists", "工作流实例[{0}]已存在"), + PROCESS_DEFINE_NOT_EXIST(50003, "process definition {0} does not exist", "工作流定义[{0}]不存在"), + PROCESS_DEFINE_NOT_RELEASE(50004, "process definition {0} not on line", "工作流定义[{0}]不是上线状态"), + PROCESS_INSTANCE_ALREADY_CHANGED(50005, "the status of process instance {0} is already {1}", "工作流实例[{0}]的状态已经是[{1}]"), + PROCESS_INSTANCE_STATE_OPERATION_ERROR(50006, "the status of process instance {0} is {1},Cannot perform {2} operation", "工作流实例[{0}]的状态是[{1}],无法执行[{2}]操作"), + SUB_PROCESS_INSTANCE_NOT_EXIST(50007, "the task belong to process instance does not exist", "子工作流实例不存在"), + PROCESS_DEFINE_NOT_ALLOWED_EDIT(50008, "process definition {0} does not allow edit", "工作流定义[{0}]不允许修改"), + PROCESS_INSTANCE_EXECUTING_COMMAND(50009, "process instance {0} is executing the command, please wait ...", "工作流实例[{0}]正在执行命令,请稍等..."), + PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE(50010, "process instance {0} is not sub process instance", "工作流实例[{0}]不是子工作流实例"), + TASK_INSTANCE_STATE_COUNT_ERROR(50011,"task instance state count error", "查询各状态任务实例数错误"), + COUNT_PROCESS_INSTANCE_STATE_ERROR(50012,"count process instance state error", "查询各状态流程实例数错误"), + COUNT_PROCESS_DEFINITION_USER_ERROR(50013,"count process definition user error", "查询各用户流程定义数错误"), + START_PROCESS_INSTANCE_ERROR(50014,"start process instance error", "运行工作流实例错误"), + EXECUTE_PROCESS_INSTANCE_ERROR(50015,"execute process instance error", "操作工作流实例错误"), + CHECK_PROCESS_DEFINITION_ERROR(50016,"check process definition error", "检查工作流实例错误"), + QUERY_RECIPIENTS_AND_COPYERS_BY_PROCESS_DEFINITION_ERROR(50017,"query recipients and copyers by process definition error", "查询收件人和抄送人错误"), + DATA_IS_NOT_VALID(50017,"data {0} not valid", "数据[{0}]无效"), + DATA_IS_NULL(50018,"data {0} is null", "数据[{0}]不能为空"), + PROCESS_NODE_HAS_CYCLE(50019,"process node has cycle", "流程节点间存在循环依赖"), + PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node {0} parameter invalid", "流程节点[{0}]参数无效"), + PROCESS_DEFINE_STATE_ONLINE(50021, "process definition {0} is already on line", "工作流定义[{0}]已上线"), + DELETE_PROCESS_DEFINE_BY_ID_ERROR(50022,"delete process definition by id error", "删除工作流定义错误"), + SCHEDULE_CRON_STATE_ONLINE(50023,"the status of schedule {0} is already on line", "调度配置[{0}]已上线"), + DELETE_SCHEDULE_CRON_BY_ID_ERROR(50024,"delete schedule by id error", "删除调度配置错误"), + BATCH_DELETE_PROCESS_DEFINE_ERROR(50025,"batch delete process definition error", "批量删除工作流定义错误"), + BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR(50026,"batch delete process definition by ids {0} error", "批量删除工作流定义[{0}]错误"), + TENANT_NOT_SUITABLE(50027,"there is not any tenant suitable, please choose a tenant available.", "没有合适的租户,请选择可用的租户"), + EXPORT_PROCESS_DEFINE_BY_ID_ERROR(50028,"export process definition by id error", "导出工作流定义错误"), + BATCH_EXPORT_PROCESS_DEFINE_BY_IDS_ERROR(50028,"batch export process definition by ids error", "批量导出工作流定义错误"), + IMPORT_PROCESS_DEFINE_ERROR(50029,"import process definition error", "导入工作流定义错误"), + + HDFS_NOT_STARTUP(60001,"hdfs not startup", "hdfs未启用"), + + /** + * for monitor + */ + QUERY_DATABASE_STATE_ERROR(70001,"query database state error", "查询数据库状态错误"), + QUERY_ZOOKEEPER_STATE_ERROR(70002,"query zookeeper state error", "查询zookeeper状态错误"), + + + + CREATE_ACCESS_TOKEN_ERROR(70010,"create access token error", "创建访问token错误"), + GENERATE_TOKEN_ERROR(70011,"generate token error", "生成token错误"), + QUERY_ACCESSTOKEN_LIST_PAGING_ERROR(70012,"query access token list paging error", "分页查询访问token列表错误"), + UPDATE_ACCESS_TOKEN_ERROR(70013,"update access token error", "更新访问token错误"), + DELETE_ACCESS_TOKEN_ERROR(70014,"delete access token error", "删除访问token错误"), + ACCESS_TOKEN_NOT_EXIST(70015, "access token not exist", "访问token不存在"), + + + COMMAND_STATE_COUNT_ERROR(80001,"task instance state count error", "查询各状态任务实例数错误"), + + QUEUE_COUNT_ERROR(90001,"queue count error", "查询队列数据错误"), + + KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error", "获取kerberos启动状态错误"), + + QUERY_PROCESS_INSTANCE_LIST_ORDER_BY_DURATION_ERROR(10113, "query process instance list order by duration error", + "分页查询工作流实例列表(耗时降序)错误"), + QUERY_PROCESS_INSTANCE_STATISTICS_ERROR(10113, "query process instance statistics error", "查询周期内实例完成情况错误") + ; + + private final int code; + private final String enMsg; + private final String zhMsg; + + private Status(int code, String enMsg, String zhMsg) { + this.code = code; + this.enMsg = enMsg; + this.zhMsg = zhMsg; + } + + public int getCode() { + return this.code; + } + + public String getMsg() { + if (Locale.SIMPLIFIED_CHINESE.getLanguage().equals(LocaleContextHolder.getLocale().getLanguage())) { + return this.zhMsg; + } else { + return this.enMsg; + } + } +} diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/interceptor/DolphinSchedulerHttpRequestUserInterceptor.java b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/interceptor/DolphinSchedulerHttpRequestUserInterceptor.java new file mode 100644 index 000000000..382717894 --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/interceptor/DolphinSchedulerHttpRequestUserInterceptor.java @@ -0,0 +1,73 @@ +package org.apache.dolphinscheduler.api.interceptor; + +import com.webank.wedatasphere.dss.standard.app.sso.plugin.filter.HttpRequestUserInterceptor; +import org.apache.commons.lang.ArrayUtils; +import org.apache.dolphinscheduler.api.controller.BaseController; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.security.Authenticator; +import org.apache.dolphinscheduler.api.service.SessionService; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; + +/** + * @author enjoyyin + * @date 2022-11-16 + * @since 1.1.1 + */ +@Component +public class DolphinSchedulerHttpRequestUserInterceptor implements HttpRequestUserInterceptor { + + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Autowired + private UserMapper userMapper; + @Autowired + private Authenticator authenticator; + @Autowired + private SessionService sessionService; + + @Override + public HttpServletRequest addUserToRequest(String user, HttpServletRequest req) { + logger.info("try to add a new session.id for DSS user {}.", user); + User userObj = userMapper.queryByUserNameAccurately(user); + if(userObj == null) { + logger.error("DSS user {} is not exists in DolphinScheduler db.", user); + throw new ServiceException(Status.USER_NOT_EXIST); + } + String sessionId = sessionService.createSession(userObj, BaseController.getClientIpAddress(req)); + Cookie cookie = new Cookie(Constants.SESSION_ID, sessionId); +// cookie.setHttpOnly(true); + cookie.setPath("/"); + logger.info("added a new session.id {} for DSS user {}.", sessionId, user); + return new HttpServletRequestWrapper(req) { + @Override + public Cookie[] getCookies() { + return (Cookie[]) ArrayUtils.add(super.getCookies(), cookie); + } + }; + } + + @Override + public boolean isUserExistInSession(HttpServletRequest req) { + return authenticator.getAuthUser(req) != null; + } + + @Override + public String getUser(HttpServletRequest req) { + User user = authenticator.getAuthUser(req); + if(user != null) { + return user.getUserName(); + } + return null; + } +} diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceMetricsService.java b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceMetricsService.java new file mode 100644 index 000000000..709c6800f --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceMetricsService.java @@ -0,0 +1,129 @@ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMetricsMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.LocalDate; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @author enjoyyin + * @date 2022-02-23 + * @since 0.5.0 + */ +@Service +public class ProcessInstanceMetricsService extends BaseDAGService { + + private static final Logger logger = LoggerFactory.getLogger(ProcessInstanceMetricsService.class); + + @Autowired + private ProjectMapper projectMapper; + @Autowired + private ProjectService projectService; + @Autowired + private UsersService usersService; + @Autowired + private ProcessInstanceMetricsMapper processInstanceMapper; + + + public Map queryProcessInstanceListOrderByDuration(User loginUser, String projectName, + Integer processDefinitionId, String executorName, String startDate, String endDate, Integer pageNo, + Integer pageSize) { + Map result = new HashMap<>(16); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status)checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + + Date start = null; + Date end = null; + try { + if (StringUtils.isNotEmpty(startDate)) { + start = DateUtils.getScheduleDate(startDate); + } + if (StringUtils.isNotEmpty(endDate)) { + end = DateUtils.getScheduleDate(endDate); + } + } catch (Exception e) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "startDate,endDate"); + return result; + } + + Page page = new Page(pageNo, pageSize); + PageInfo pageInfo = new PageInfo(pageNo, pageSize); + int executorId = usersService.getUserIdByName(executorName); + + IPage processInstanceList = processInstanceMapper.queryProcessInstanceListOrderByDuration(page, + project.getId(), processDefinitionId, executorId, start, end); + + List processInstances = processInstanceList.getRecords(); + + for (ProcessInstance processInstance : processInstances) { + processInstance.setDuration( + DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime())); + User executor = usersService.queryUser(processInstance.getExecutorId()); + if (null != executor) { + processInstance.setExecutorName(executor.getUserName()); + } + } + + pageInfo.setTotalCount((int)processInstanceList.getTotal()); + pageInfo.setLists(processInstances); + result.put(Constants.DATA_LIST, pageInfo); + putMsg(result, Status.SUCCESS); + return result; + } + + public Map queryProcessInstanceStatistics(User loginUser, String projectName, + Integer processDefinitionId, String executorName, List dateList, int state, int step) { + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status)checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + + Map dataMap = new HashMap<>(8); + int executorId = usersService.getUserIdByName(executorName); + for (String startDate : dateList) { + LocalDate start = LocalDate.parse(startDate); + LocalDate end = start.plusDays(1); + List processInstanceList = processInstanceMapper.queryProcessInstanceListByStartTime( + project.getId(), processDefinitionId, executorId, start, end, state); + + List counts = new ArrayList<>(Collections.nCopies((int)Math.ceil(24.0 / step), 0)); + Map> map = processInstanceList.stream() + .collect(Collectors.groupingBy( + processInstance -> DateUtils.getHourIndex(processInstance.getStartTime()) / step)); + map.forEach((k, v) -> counts.set(k, v.size())); + + dataMap.put(startDate, counts); + } + + Map result = new HashMap<>(4); + result.put(Constants.DATA_LIST, dataMap); + putMsg(result, Status.SUCCESS); + return result; + } + + +} diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMetricsMapper.java b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMetricsMapper.java new file mode 100644 index 000000000..77b46c5de --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMetricsMapper.java @@ -0,0 +1,27 @@ +package org.apache.dolphinscheduler.dao.mapper; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.ibatis.annotations.Param; + +import java.time.LocalDate; +import java.util.Date; +import java.util.List; + +/** + * @author enjoyyin + * @date 2022-02-23 + * @since 0.5.0 + */ +public interface ProcessInstanceMetricsMapper { + + IPage queryProcessInstanceListOrderByDuration(Page page, + @Param("projectId") int projectId, @Param("processDefinitionId") Integer processDefinitionId, + @Param("executorId") Integer executorId, @Param("startTime") Date startTime, @Param("endTime") Date endTime); + + List queryProcessInstanceListByStartTime(@Param("projectId") int projectId, + @Param("processDefinitionId") Integer processDefinitionId, @Param("executorId") Integer executorId, + @Param("startTime") LocalDate start, @Param("endTime") LocalDate end, @Param("state") int state); + +} diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMetricsMapper.xml b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMetricsMapper.xml new file mode 100644 index 000000000..89772fcf6 --- /dev/null +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMetricsMapper.xml @@ -0,0 +1,58 @@ + + + + + + + + \ No newline at end of file diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/bin/dss-dolphinscheduler-client.sh b/plugins/dolphinscheduler/dss-dolphinscheduler-client/bin/dss-dolphinscheduler-client.sh new file mode 100644 index 000000000..7494b845e --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/bin/dss-dolphinscheduler-client.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# description: dss-dolphinscheduler-client start cmd +# + +# get log directory +cd `dirname $0` +cd .. +DSS_DOLPHINSCHEDULER_CLIENT_HOME=`pwd` + +NODE_NAME=$1 + +# set DSS_CONF_DIR +if [ "$DSS_DOLPHINSCHEDULER_CLIENT_CONF_DIR" = "" ]; then + export DSS_DOLPHINSCHEDULER_CLIENT_CONF_DIR=$DSS_DOLPHINSCHEDULER_CLIENT_HOME/conf +fi + +SERVER_SUFFIX="dss-dophinscheduler-client-$NODE_NAME" +## set log +if [ "$DSS_DOLPHINSCHEDULER_CLIENT_LOG_DIR" = "" ]; then + export DSS_DOLPHINSCHEDULER_CLIENT_LOG_DIR="$DSS_DOLPHINSCHEDULER_CLIENT_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_DOLPHINSCHEDULER_CLIENT_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="256M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.DSSDolphinSchedulerClient + +## conf dir +export SERVER_CONF_PATH=$DSS_DOLPHINSCHEDULER_CLIENT_CONF_DIR + +## commons lib +export DSS_DOLPHINSCHEDULER_CLIENT_COMMONS_LIB="$DSS_DOLPHINSCHEDULER_CLIENT_HOME/lib" + +if [ ! -r "$DSS_DOLPHINSCHEDULER_CLIENT_COMMONS_LIB" ] ; then + echo "the dss-dolphinscheduler-client lib '$DSS_DOLPHINSCHEDULER_CLIENT_COMMONS_LIB' is not exists." + exit 1 +fi + +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_DOLPHINSCHEDULER_CLIENT_COMMONS_LIB/* + +which java +r=$? + +if [ $r == 0 ]; then + echo "command: java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS" + java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS +else + echo "command: $JAVA_HOME/java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS" + $JAVA_HOME/java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS +fi \ No newline at end of file diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml b/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml new file mode 100644 index 000000000..2f9180eee --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml @@ -0,0 +1,105 @@ + + + + + + dss-plugin-dolphinscheduler + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-dolphinscheduler-client + + + + com.webank.wedatasphere.dss + dss-linkis-node-execution + ${dss.version} + + + + + org.slf4j + slf4j-simple + 1.7.30 + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + dss-dolphinscheduler-client + false + false + + jar-with-dependencies + + + + + + com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.DSSDolphinSchedulerClient + + + + + src/main/assembly/distribution.xml + + + + + + + \ No newline at end of file diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/assembly/distribution.xml b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/assembly/distribution.xml new file mode 100644 index 000000000..a73841759 --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/assembly/distribution.xml @@ -0,0 +1,73 @@ + + + + dss-dolphinscheduler-client + + zip + + true + dss-dolphinscheduler-client + + + + + + lib + true + true + false + true + true + + + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/resources/linkis.properties b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/resources/linkis.properties new file mode 100644 index 000000000..24a0f7dab --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/resources/linkis.properties @@ -0,0 +1,5 @@ +wds.linkis.gateway.url= +wds.linkis.flow.job.creator.v1=DolphinScheduler + +#wds.linkis.spark.engine.version=2.4.3 +#wds.linkis.hive.engine.version=2.3.3 \ No newline at end of file diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/resources/log4j2.xml b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/resources/log4j2.xml new file mode 100644 index 000000000..e4ebb9f95 --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/resources/log4j2.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/DSSDolphinSchedulerClient.scala b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/DSSDolphinSchedulerClient.scala new file mode 100644 index 000000000..9fdee6ad5 --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/DSSDolphinSchedulerClient.scala @@ -0,0 +1,113 @@ +package com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client + +import java.util + +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration +import com.webank.wedatasphere.dss.linkis.node.execution.execution.impl.LinkisNodeExecutionImpl +import com.webank.wedatasphere.dss.linkis.node.execution.job.{JobTypeEnum, LinkisJob} +import com.webank.wedatasphere.dss.linkis.node.execution.listener.LinkisExecutionListener +import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog +import com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.conf.LinkisJobTypeConf +import com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.job.DolphinSchedulerJobBuilder +import org.apache.commons.lang.StringUtils +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.utils.{Logging, Utils} + +object DSSDolphinSchedulerClient extends Logging { + + def main(args: Array[String]): Unit = { + + val dsVersion = CommonVars("DS_VERSION", "").getValue + info(s"The DolphinScheduler Version => $dsVersion.") + + val logObj: LinkisJobExecutionLog = if(StringUtils.isNotBlank(dsVersion) && dsVersion.startsWith("2.")) { + new LinkisJobExecutionLog { + override def info(message: Object, t: Throwable): Unit = + if(message != null) logger.info(message.toString, t) else if(t != null) logger.info(null, t) + override def warn(message: Object, t: Throwable): Unit = + if(message != null) logger.warn(message.toString, t) else if(t != null) logger.warn(null, t) + override def error(message: Object, t: Throwable): Unit = + if(message != null) logger.error(message.toString, t) else if(t != null) logger.error(null, t) + } + } else { + warn("For the lower DolphinScheduler versions(< 2.0), we will parse the yarn applicationId to application__\\d+_\\d+.") + warn("Why? Because the lower DolphinScheduler versions will block the thread until the yarn application is completed.") + // For more information, please visit the class AbstractCommandExecutor in DolphinScheduler (line 222 ~ 231). + new LinkisJobExecutionLog { + private implicit def parseYarnApplication(message: Object): String = message match { + case str: String if str.contains(" application_") => + str.replaceAll(" application_", " application__") + case str: String => str + case obj if obj != null => obj.toString + case _ => "" + } + override def info(message: Object, t: Throwable): Unit = logger.info(message, t) + override def warn(message: Object, t: Throwable): Unit = logger.warn(message, t) + override def error(message: Object, t: Throwable): Unit = logger.error(message, t) + } + } + + val jobProps = new util.HashMap[String, String] + val getAndSet = (fromKey: String, toKey: String) => { + val value = CommonVars(fromKey, "").getValue + if (StringUtils.isNotBlank(value)){ + logObj.info(s"$toKey => $value.") + jobProps.put(toKey, value) + } + } + getAndSet("LINKIS_TYPE", LinkisJobTypeConf.LINKIS_TYPE) + getAndSet("LINKIS_VERSION", LinkisJobTypeConf.LINKIS_VERSION) + getAndSet("RUN_DATE", LinkisJobTypeConf.RUN_DATE) + getAndSet("CONTEXT_ID", LinkisJobExecutionConfiguration.FLOW_CONTEXTID) + getAndSet("LINKIS_GATEWAY_URL", LinkisJobExecutionConfiguration.LINKIS_URL_1_X.key) + getAndSet("PROXY_USER", LinkisJobTypeConf.PROXY_USER) + getAndSet("JOB_SOURCE", LinkisJobTypeConf.JOB_SOURCE) + getAndSet("JOB_PARAMS", LinkisJobTypeConf.JOB_PARAMS) + getAndSet("JOB_COMMAND", LinkisJobTypeConf.JOB_COMMAND) + getAndSet("JOB_RESOURCES", LinkisJobTypeConf.JOB_RESOURCES) + getAndSet("FLOW_RESOURCES", LinkisJobTypeConf.FLOW_RESOURCES) + getAndSet("FLOW_PROPERTIES", LinkisJobTypeConf.FLOW_PROPERTIES) + getAndSet("JOB_LABELS", LinkisJobTypeConf.JOB_LABELS) + val job = new DolphinSchedulerJobBuilder(jobProps).build() match { + case linkisJob: LinkisJob => linkisJob + } + job.setLogObj(logObj) + job.getJobType match { + case JobTypeEnum.EmptyJob => + logObj.warn("This node is empty node, just do nothing and return.") + return + case _ => + } + + val execution = LinkisNodeExecutionImpl.getLinkisNodeExecution + execution.runJob(job) + + Utils.addShutdownHook { + if(!execution.isCompleted(job)) { + logObj.info(s"try to kill job ${job.getJobName}, since a shutdown signal is called.") + execution.cancel(job) + } + } + execution.waitForComplete(job) + + Utils.tryAndWarn { + val endLog = LinkisNodeExecutionImpl.getLinkisNodeExecution.getLog(job) + logObj.info(endLog) + } + + LinkisNodeExecutionImpl.getLinkisNodeExecution match { + case listener: LinkisExecutionListener => + listener.onStatusChanged(null, LinkisNodeExecutionImpl.getLinkisNodeExecution.getState(job), job) + } + + val resultSize = Utils.tryAndWarn(LinkisNodeExecutionImpl.getLinkisNodeExecution.getResultSize(job)) + if(resultSize > 0) (0 until resultSize).foreach{ index => + val resultLine = LinkisNodeExecutionImpl.getLinkisNodeExecution.getResult(job, index, LinkisJobExecutionConfiguration.RESULT_PRINT_SIZE.getValue(jobProps)) + logObj.info(s"The content of the ${index + 1}th resultset is : $resultLine.") + } + logObj.info("\n") + logObj.info("-----------------------------------------------------------") + logObj.info("Finished to execute job.") + } + +} diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/conf/LinkisJobTypeConf.scala b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/conf/LinkisJobTypeConf.scala new file mode 100644 index 000000000..dc41386c8 --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/conf/LinkisJobTypeConf.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.conf + + +object LinkisJobTypeConf { + + val LINKIS_TYPE = "linkistype" + val RUN_DATE = "run_date" + val JOB_ID = "task.instance.key.str" + + val JOB_COMMAND = "jobCommand" + val JOB_PARAMS = "jobParams" + val JOB_SOURCE = "jobSource" + val JOB_RESOURCES = "jobResources" + val JOB_LABELS = "jobLabels" + + val FLOW_RESOURCES = "flowResources" + val FLOW_PROPERTIES = "flowProperties" + + + val PROXY_USER = "proxy.user" + val FLOW_SUBMIT_USER = "submit.user" + + val LINKIS_VERSION = "linkis.version" + +} diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerAppConnLinkisJob.scala b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerAppConnLinkisJob.scala new file mode 100644 index 000000000..0379cb4bd --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerAppConnLinkisJob.scala @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.job + +import com.webank.wedatasphere.dss.linkis.node.execution.job.AbstractAppConnLinkisJob +import com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.conf.LinkisJobTypeConf +import org.apache.commons.lang3.StringUtils + +class DolphinSchedulerAppConnLinkisJob extends AbstractAppConnLinkisJob { + override def getSubmitUser: String = if (StringUtils.isBlank(getJobProps.get(LinkisJobTypeConf.FLOW_SUBMIT_USER))) + getJobProps.get(LinkisJobTypeConf.PROXY_USER) else getJobProps.get(LinkisJobTypeConf.FLOW_SUBMIT_USER) + + override def getUser: String = getJobProps.get(LinkisJobTypeConf.PROXY_USER) + + override def getJobName: String = getJobProps.get(LinkisJobTypeConf.JOB_ID, "test") +} diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerCommonLinkisJob.scala b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerCommonLinkisJob.scala new file mode 100644 index 000000000..9532a895c --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerCommonLinkisJob.scala @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.job + +import com.webank.wedatasphere.dss.linkis.node.execution.job.AbstractCommonLinkisJob +import com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.conf.LinkisJobTypeConf + +class DolphinSchedulerCommonLinkisJob extends AbstractCommonLinkisJob { + override def getSubmitUser: String = if (getJobProps.get(LinkisJobTypeConf.FLOW_SUBMIT_USER) == null) getJobProps.get(LinkisJobTypeConf.PROXY_USER) else getJobProps.get(LinkisJobTypeConf.FLOW_SUBMIT_USER) + + override def getUser: String = getJobProps.get(LinkisJobTypeConf.PROXY_USER) + + override def getJobName: String = getJobProps.getOrDefault(LinkisJobTypeConf.JOB_ID, "test") +} diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerJobBuilder.scala b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerJobBuilder.scala new file mode 100644 index 000000000..f039985b7 --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/src/main/scala/com/webank/wedatasphere/dss/plugins/dolphinscheduler/linkis/client/job/DolphinSchedulerJobBuilder.scala @@ -0,0 +1,93 @@ +package com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.job + +import java.util +import java.util.{Map => JMap} + +import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration +import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource +import com.webank.wedatasphere.dss.linkis.node.execution.job.{Builder, CommonLinkisJob, Job, LinkisJob} +import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils +import com.webank.wedatasphere.dss.plugins.dolphinscheduler.linkis.client.conf.LinkisJobTypeConf +import org.apache.commons.lang.StringUtils +import org.apache.linkis.common.utils.JsonUtils + +import scala.collection.JavaConverters._ + +class DolphinSchedulerJobBuilder(jobProps: JMap[String, String]) extends Builder { + + override protected def getJobType: String = jobProps.get(LinkisJobTypeConf.LINKIS_TYPE) + + override protected def creatLinkisJob(isLinkisType: Boolean): LinkisJob = { + val job = if (isLinkisType) new DolphinSchedulerCommonLinkisJob else new DolphinSchedulerAppConnLinkisJob + job.setJobProps(jobProps) + job + } + + override protected def fillJobInfo(job: Job): Unit = { + job.setCode(jobProps.get(LinkisJobTypeConf.JOB_COMMAND)) + job.setParams(new java.util.HashMap[String, Object]()) + job.setRuntimeParams(new java.util.HashMap[String, Object]) + job.setJobProps(jobProps) + } + + override protected def fillLinkisJobInfo(linkisJob: LinkisJob): Unit = { + val jobParams = jobProps.get(LinkisJobTypeConf.JOB_PARAMS) + if(StringUtils.isNotBlank(jobParams)){ + val paramsMap = JsonUtils.jackson.readValue(jobParams, classOf[util.Map[String, Object]]) + linkisJob.setConfiguration(getConfiguration(paramsMap)) + linkisJob.setVariables(getVariables(paramsMap)) + } + val runDate = new util.HashMap[String, Object] + runDate.put("run_date", jobProps.get(LinkisJobTypeConf.RUN_DATE)) + linkisJob.setVariables(runDate) + + val source = getSource + linkisJob.setSource(source) + linkisJob.getRuntimeParams.put("nodeName", source.get("nodeName")) + linkisJob.getRuntimeParams.put("labels", jobProps.get(LinkisJobTypeConf.JOB_LABELS)) + } + + override protected def getContextID(job: Job): String = jobProps.get(LinkisJobExecutionConfiguration.FLOW_CONTEXTID) + + override protected def fillCommonLinkisJobInfo(commonLinkisJob: CommonLinkisJob): Unit = { + val jobResources = jobProps.get(LinkisJobTypeConf.JOB_RESOURCES) + if(StringUtils.isNotBlank(jobResources)){ + commonLinkisJob.setJobResourceList(LinkisJobExecutionUtils.getResourceListByJson(jobResources)) + } + val flowNameAndResources = new util.HashMap[String, util.List[BMLResource]] + if(jobProps.containsKey(LinkisJobTypeConf.FLOW_RESOURCES)) { + flowNameAndResources.put(getSource.get("flowName") + LinkisJobExecutionConfiguration.RESOURCES_NAME, + LinkisJobExecutionUtils.getResourceListByJson(jobProps.get(LinkisJobTypeConf.FLOW_RESOURCES))) + } + commonLinkisJob.setFlowNameAndResources(flowNameAndResources) + } + + private def getSource: JMap[String, String] = { + val jobSource = jobProps.get(LinkisJobTypeConf.JOB_SOURCE) + if(StringUtils.isNotBlank(jobSource)) { + JsonUtils.jackson.readValue(jobSource, classOf[util.Map[String, String]]) + } else new util.HashMap[String, String] + } + + private def getVariables(params: util.Map[String, Object]): util.Map[String, Object] = { + val variable = params.get("variable") match { + case variables: util.Map[String, Object] => + variables + case _ => new util.HashMap[String, AnyRef] + } + val flowProperties = jobProps.get(LinkisJobTypeConf.FLOW_PROPERTIES) + if(StringUtils.isNotBlank(flowProperties)) + JsonUtils.jackson.readValue(flowProperties, classOf[util.List[util.Map[String, Object]]]) + .asScala.foreach(variable.putAll(_)) + variable + } + + private def getConfiguration(params: util.Map[String, Object]): util.Map[String, Object] = { + params.get("configuration") match { + case configuration: util.Map[String, Object] => + configuration + case _ => new util.HashMap[String, AnyRef] + } + } + +} diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml b/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml new file mode 100644 index 000000000..c88da378f --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml @@ -0,0 +1,70 @@ + + + + + + dss-plugin-dolphinscheduler + com.webank.wedatasphere.dss + 1.1.1 + + + 4.0.0 + + dss-dolphinscheduler-token + + + + com.webank.wedatasphere.dss + dss-framework-project-server + ${dss.version} + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + linkis-common + org.apache.linkis + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + \ No newline at end of file diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-token/src/main/java/com/webank/wedatasphere/dss/dolphinscheduler/token/restful/DolphinSchedulerTokenRestfulApi.java b/plugins/dolphinscheduler/dss-dolphinscheduler-token/src/main/java/com/webank/wedatasphere/dss/dolphinscheduler/token/restful/DolphinSchedulerTokenRestfulApi.java new file mode 100644 index 000000000..b219172a4 --- /dev/null +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-token/src/main/java/com/webank/wedatasphere/dss/dolphinscheduler/token/restful/DolphinSchedulerTokenRestfulApi.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.dolphinscheduler.token.restful; + +import com.webank.wedatasphere.dss.appconn.core.ext.OptionalAppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; +import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; +import com.webank.wedatasphere.dss.standard.app.structure.optional.OptionalOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; + +/** + * @author enjoyyin + * @date 2022-03-18 + * @since 0.5.0 + */ +@RequestMapping(path = "/dss/framework/project", produces = {"application/json"}) +@RestController +public class DolphinSchedulerTokenRestfulApi { + + public static final String DOLPHIN_SCHEDULER_APP_CONN_NAME = "dolphinscheduler"; + + @RequestMapping(path ="/ds/token", method = RequestMethod.GET) + public Message dsApiServiceTokenCreate(HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + SchedulerAppConn schedulerAppConn = (SchedulerAppConn) AppConnManager.getAppConnManager().getAppConn(DOLPHIN_SCHEDULER_APP_CONN_NAME); + AppInstance appInstance = schedulerAppConn.getAppDesc().getAppInstances().get(0); + OptionalOperation optionalOperation = ((OptionalAppConn) schedulerAppConn).getOrCreateOptionalStandard() + .getOptionalService(appInstance).getOptionalOperation("getToken"); + ResponseRef responseRef = optionalOperation.apply(new StructureRequestRefImpl().setUserName(userName)); + return Message.ok().data("token", responseRef.getValue("token")) + .data("expire_time", responseRef.getValue("expireTime")); + } +} diff --git a/plugins/dolphinscheduler/pom.xml b/plugins/dolphinscheduler/pom.xml new file mode 100644 index 000000000..049de359c --- /dev/null +++ b/plugins/dolphinscheduler/pom.xml @@ -0,0 +1,42 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + ../../pom.xml + + 4.0.0 + + dss-plugin-dolphinscheduler + pom + + + dolphinscheduler-prod-metrics + dss-dolphinscheduler-token + dss-dolphinscheduler-client + + + + 1.3.9 + + + \ No newline at end of file diff --git a/plugins/linkis/dss-gateway-support/pom.xml b/plugins/linkis/dss-gateway-support/pom.xml new file mode 100644 index 000000000..06616b65e --- /dev/null +++ b/plugins/linkis/dss-gateway-support/pom.xml @@ -0,0 +1,55 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.1.1 + + 4.0.0 + + dss-gateway-support + + + org.apache.linkis + linkis-gateway-server-support + ${linkis.version} + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + + \ No newline at end of file diff --git a/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayConfiguration.scala b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayConfiguration.scala new file mode 100644 index 000000000..c62734f7b --- /dev/null +++ b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayConfiguration.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.gateway.parser + +import org.apache.linkis.common.conf.CommonVars + + +object DSSGatewayConfiguration { + val DSS_SPRING_NAME = CommonVars("wds.linkis.dss.name", "dss-server") + + val DSS_URL_LABEL_PREFIX = CommonVars("wds.dss.gateway.url.prefix.name", "labels") + + val DSS_URL_ROUTE_LABEL_PREFIX = CommonVars("wds.dss.gateway.url.prefix.name", "labelsRoute") + + val DSS_URL_APPCONNS = CommonVars("wds.dss.gateway.url.appconns", "visualis") + + +} diff --git a/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayParser.scala b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayParser.scala new file mode 100644 index 000000000..9b71d64ec --- /dev/null +++ b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayParser.scala @@ -0,0 +1,209 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.gateway.parser + +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.gateway.exception.TooManyServiceException +import org.apache.linkis.gateway.http.GatewayContext +import org.apache.linkis.gateway.springcloud.SpringCloudGatewayConfiguration.{API_URL_PREFIX, normalPath} +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.protocol.constants.TaskConstant +import org.apache.linkis.rpc.sender.SpringCloudFeignConfigurationCache +import org.apache.linkis.server.BDPJettyServerHelper +import org.springframework.stereotype.Component + +import java.util +import scala.collection.JavaConversions._ + + +@Component +class DSSGatewayParser extends AbstractGatewayParser { + + val appConns = DSSGatewayConfiguration.DSS_URL_APPCONNS.getValue.split(",") + + override def shouldContainRequestBody(gatewayContext: GatewayContext): Boolean = { + var contentType = gatewayContext.getRequest.getHeaders.get("Content-Type") + if(null == contentType) { + contentType = gatewayContext.getRequest.getHeaders.get("content-type") + } + + if(contentType != null && contentType.nonEmpty + && contentType(0).contains("form-data")){ + logger.info("DSS gateway get request type is form-data") + return false + } + + gatewayContext.getRequest.getRequestURI match { + case DSSGatewayParser.DSS_URL_DEFAULT_REGEX(_, _) => true + case DSSGatewayParser.DSS_URL_REGEX(_, _,_) => true + case DSSGatewayParser.APPCONN_URL_DEFAULT_REGEX(_,appconn, _) if appConns.contains(appconn) => true + case _ => false + } + } + + override def parse(gatewayContext: GatewayContext): Unit = gatewayContext.getRequest.getRequestURI match { + + case DSSGatewayParser.DSS_URL_REGEX(version, firstName, secondName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = "dss-" + firstName + "-" + secondName + "-server" + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService("dss/" + firstName + "/" + secondName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info("Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI) + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null)) + } + case DSSGatewayParser.DSS_URL_DEFAULT_REGEX(version, firstName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = "dss-" + firstName + "-server" + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService("dss/" + firstName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info("Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI) + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null)) + } + case DSSGatewayParser.APPCONN_URL_DEFAULT_REGEX(version, serverName, _) if appConns.contains(serverName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = serverName + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService(tmpServerName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info("Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI) + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null)) + } + case _ => + } + + private def getServiceNameFromLabel(gatewayContext: GatewayContext, tmpServiceName: String): String = { + var requestUrlLabels = gatewayContext.getRequest.getQueryParams.getOrDefault(DSSGatewayConfiguration.DSS_URL_LABEL_PREFIX.getValue, null) + if(requestUrlLabels == null) requestUrlLabels = gatewayContext.getRequest.getQueryParams.getOrDefault(DSSGatewayConfiguration.DSS_URL_ROUTE_LABEL_PREFIX.getValue,null) + logger.info("Get ServiceName From Label and method is "+ gatewayContext.getRequest.getMethod.toString+",and urlLabels is "+requestUrlLabels) + val requestMethod = gatewayContext.getRequest.getMethod.toLowerCase + if (requestUrlLabels == null && (requestMethod.equals("post") || requestMethod.equals("put") || requestMethod.equals("delete"))) { + val requestBody = Option(gatewayContext.getRequest.getRequestBody) + val routeLabelList = new util.ArrayList[RouteLabel]() + + requestBody match { + //todo form-data resolve + case Some(body) => + val labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory + val json = BDPJettyServerHelper.gson.fromJson(body, classOf[java.util.Map[String, Object]]) + val labels: util.List[Label[_]] = json.get(TaskConstant.LABELS) match { + case map: util.Map[String, Object] => labelBuilderFactory.getLabels(map) + case map: util.Map[String, Any] => labelBuilderFactory.getLabels(map.asInstanceOf) + case _ => new util.ArrayList[Label[_]]() + } + labels.filter(label => label.isInstanceOf[RouteLabel]).foreach(label => { + routeLabelList.add(label.asInstanceOf[RouteLabel]) + }) + + case _ => null + } + val labelNameList = routeLabelList.map(routeLabel => routeLabel.getStringValue).toList + if(labelNameList != null && labelNameList.size>0) { + genServiceNameByDSSLabel(labelNameList, tmpServiceName) + }else if(null != requestUrlLabels){ + genServiceNameByDSSLabel(requestUrlLabels.toList, tmpServiceName) + } else tmpServiceName + + } else { + if(requestUrlLabels != null) { + genServiceNameByDSSLabel(requestUrlLabels.toList, tmpServiceName) + }else tmpServiceName + } + } + + private def genServiceNameByDSSLabel(labelList: List[String], tmpServiceName: String): String = { + var resultName = tmpServiceName + if (null != labelList && labelList.size > 0) { + val labelNameList = labelList(0).replace(" ", "").split(",").toList + if (labelNameList.size > 0) { + if (labelNameList.find(name => name.equalsIgnoreCase("dev")).isDefined) { + resultName = tmpServiceName + "-dev" + } else if (labelNameList.find(name => name.equalsIgnoreCase("prod")).isDefined) { + resultName = tmpServiceName + "-prod" + } else if (labelNameList.find(name => name.equalsIgnoreCase("test")).isDefined){ + resultName = tmpServiceName + "-test" + }else{ + resultName = tmpServiceName + } + } + } + resultName + } + + + private def findCommonService(parsedServiceId: String, tmpServerName: String) = findService(parsedServiceId, tmpServerName, services => { + val errorMsg = new TooManyServiceException(s"Cannot find a correct serviceId for parsedServiceId $parsedServiceId, service list is: " + services) + warn("", errorMsg) + throw errorMsg + }) + + protected def findService(parsedServiceId: String, tmpServerName: String, tooManyDeal: List[String] => Option[String]): Option[String] = { + val findIt: (String => Boolean) => Option[String] = op => { + val services = SpringCloudFeignConfigurationCache.getDiscoveryClient + .getServices.filter(op).toList + if (services.length == 1) Some(services.head) + else if (services.length > 1) tooManyDeal(services) + else None + } + //通过匹配到最多的url中的path进行路由,如/dss/framework/workspace/ 会匹配到 dss-framework-workspace-server 而不是 dss-server + // 如果产生了相等的情况,则按照短的service名字为准 比如/dss/getProject, + // 我们可能会匹配到dss-server以及 dss-framework-workspace-server,则选择短名称的dss-server + val findMostCorrect: (String => (String, Int)) => Option[String] = { op => { + val serviceMap = SpringCloudFeignConfigurationCache.getDiscoveryClient + .getServices.map(op).toMap + var count = 0 + var retService: Option[String] = None + serviceMap.foreach { + case (k, v) => if (v > count) { + count = v + retService = Some(k) + } else if (retService.isDefined && v == count && k.length < retService.get.length) { + retService = Some(k) + } + } + retService + } + } + val lowerServiceId = parsedServiceId.toLowerCase + val serverName = tmpServerName.toLowerCase + //findIt(_.toLowerCase == lowerServiceId).orElse(findIt(_.toLowerCase.contains(lowerServiceId))) + findIt(_.toLowerCase == serverName).orElse(findMostCorrect(service => { + (service, lowerServiceId.split("/").count(word => service.contains(word))) + })) + } + + +} + +object DSSGatewayParser { + val DSS_HEADER = normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/dss/" + val DSS_URL_REGEX = (DSS_HEADER + "([^/]+)/" + "([^/]+)/.+").r + val DSS_URL_DEFAULT_REGEX = (DSS_HEADER + "([^/]+).+").r + + val APPCONN_HEADER = normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/([^/]+)/" + val APPCONN_URL_DEFAULT_REGEX = (APPCONN_HEADER + "([^/]+).+").r + +} diff --git a/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSRouteLabelParser.scala b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSRouteLabelParser.scala new file mode 100644 index 000000000..a56dbe594 --- /dev/null +++ b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSRouteLabelParser.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.gateway.parser + +import org.apache.linkis.gateway.http.GatewayContext +import org.apache.linkis.gateway.ujes.route.label.RouteLabelParser +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.springframework.stereotype.Component + +import java.util + + +@Component +class DSSRouteLabelParser extends RouteLabelParser{ + override def parse(gatewayContext: GatewayContext): util.List[RouteLabel] = { + new util.ArrayList[RouteLabel]() + } +} diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000..80ffb18d1 --- /dev/null +++ b/pom.xml @@ -0,0 +1,251 @@ + + + + + 4.0.0 + pom + com.webank.wedatasphere.dss + dss + 1.1.1 + + + dss-commons + dss-standard + dss-orchestrator + dss-appconn + dss-framework + dss-apps/dss-apiservice-server + dss-apps/dss-scriptis-server + dss-apps/dss-user-guide + plugins/azkaban/linkis-jobtype + plugins/linkis/dss-gateway-support + assembly + plugins/dolphinscheduler + + + + 1.1.1 + 1.1.1 + 2.11.12 + 1.8 + 3.3.3 + 2.8.5 + 2.11.3 + 1.9.13 + 3.1.1 + 4.5.4 + 4.5.4 + 1.9.4 + UTF-8 + 5.2.22.RELEASE + 2.1.2 + 2.3.7.RELEASE + 2.2.6.RELEASE + 3.1.1 + 3.8.1 + 2.6 + 1.4.19 + 2.30.1 + + + + + + org.scala-lang + scala-library + ${scala.version} + + + org.scala-lang + scala-compiler + ${scala.version} + + + org.scala-lang + scala-reflect + ${scala.version} + + + org.scala-lang + scalap + ${scala.version} + + + commons-lang + commons-lang + ${commons.lang.version} + + + org.apache.linkis + linkis-common + ${linkis.version} + + + com.google.code.gson + gson + ${gson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + ${org.codehaus.jackson.version} + + + org.apache.commons + commons-math3 + ${commons.math.version} + + + xstream + com.thoughtworks.xstream + ${xstream.version} + + + + + + + releases + Releases + https://127.0.0.1/repository/maven-releases/ + + + snapshots + Snapshot + https://127.0.0.1/repository/maven-snapshots/ + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4.1 + + + enforce-versions + + enforce + + + + + ${maven.version} + + + ${java.version} + + + + org.jboss.netty + + true + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.5.1 + + ${jdk.compile.version} + ${jdk.compile.version} + UTF-8 + + + + org.apache.maven.plugins + maven-site-plugin + 3.3 + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + eclipse-add-source + + add-source + + + + scala-compile-first + process-resources + + compile + + + + scala-test-compile-first + process-test-resources + + testCompile + + + + attach-scaladocs + verify + + doc-jar + + + + + ${scala.version} + all + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.6 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + \ No newline at end of file diff --git a/sbin/common.sh b/sbin/common.sh new file mode 100644 index 000000000..1fde09493 --- /dev/null +++ b/sbin/common.sh @@ -0,0 +1,188 @@ +#!/bin/sh +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#Actively load user env +source ~/.bash_profile + +export local_host="`hostname --fqdn`" + +#ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}') + +function isLocal(){ + if [ "$1" == "127.0.0.1" ];then + return 0 + elif [ "$1" == "" ]; then + return 0 + elif [ "$1" == "localhost" ]; then + return 0 + elif [ "$1" == $local_host ]; then + return 0 + #elif [ "$1" == $ipaddr ]; then + # return 0 + fi + return 1 +} + +function executeCMD(){ + isLocal $1 + flag=$? + if [ $flag == "0" ];then + echo "Is local execution:$2" + eval $2 + else + echo "Is remote execution:$2" + ssh -p $SSH_PORT $1 $2 + fi + +} +function copyFile(){ + isLocal $1 + flag=$? + src=$2 + dest=$3 + if [ $flag == "0" ];then + echo "Is local cp " + cp -r "$src" "$dest" + else + echo "Is remote cp " + scp -r -P $SSH_PORT "$src" $1:"$dest" + fi + +} + +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 +else + echo "Succeed to" + $1 +fi +} + + +function start() +{ + echo "Start to check whether the $SERVER_NAME is running" + if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "$SERVER_NAME is already running." + exit 1 + fi + fi + export SERVER_START_BIN=$DSS_HOME/sbin/ext/$SERVER_NAME + if [[ ! -f "${SERVER_START_BIN}" ]]; then + echo "The $SERVER_NAME is wrong or the corresponding startup script does not exist: " + echo "$SERVER_START_BIN" + exit 1 + else + echo "Start to start server, startup script: $SERVER_START_BIN" + sh $SERVER_START_BIN + fi +} + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + + +function stop() +{ + if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" + else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi + fi +} + +function restart() +{ + stop + sleep 2 + start +} + +status() +{ + if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is stopped" + else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + echo "server $SERVER_NAME is running." + fi + fi +} + +function setServerName(){ + if [[ $PROJECT_NAME == *"project"* ]]; then + SERVER_NAME=dss-framework-project-server + elif [[ $PROJECT_NAME == *"orchestrator"* ]]; then + SERVER_NAME=dss-framework-orchestrator-server + elif [[ $PROJECT_NAME == *"apiservice"* ]]; then + SERVER_NAME=dss-apiservice-server + elif [[ $PROJECT_NAME == *"scriptis"* ]]; then + SERVER_NAME=dss-scriptis-server + elif [[ $PROJECT_NAME == *"workflow"* ]]; then + SERVER_NAME=dss-workflow-server + elif [[ $PROJECT_NAME == *"execution"* ]]; then + SERVER_NAME=dss-flow-execution-server + elif [[ $PROJECT_NAME == *"data-api"* ]]; then + SERVER_NAME=dss-data-api-server + elif [[ $PROJECT_NAME == *"governance"* ]]; then + SERVER_NAME=dss-data-governance-server + elif [[ $PROJECT_NAME == *"guide"* ]]; then + SERVER_NAME=dss-guide-server + else + echo "please input: sh dss-daemon.sh [start,restart,stop] [server name]; for example : sh dss-daemon.sh restart project " + echo "server name : project、orchestrator、apiservice、scriptis、workflow、execution、data-api、governance、guide" + exit 1 + fi +} + diff --git a/sbin/dss-daemon.sh b/sbin/dss-daemon.sh new file mode 100644 index 000000000..6f60feed8 --- /dev/null +++ b/sbin/dss-daemon.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# description: Starts and stops Server +# + +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +function print_usage(){ + echo "Usage: dss-daemon [start | stop | restart | status] [serverName]" + echo " serverName The service name of the operation" + echo "Most commands print help when invoked w/o parameters." +} + +if [ $# != 2 ]; then + print_usage + exit 2 +fi + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +# get pid directory +if [ "$DSS_PID_DIR" = "" ]; then + export DSS_PID_DIR="$DSS_HOME/pid" +fi +if [ ! -w "$DSS_PID_DIR" ] ; then + mkdir -p "$DSS_PID_DIR" +fi +source $DSS_HOME/sbin/common.sh +source $DSS_CONF_DIR/config.sh +typeset -l PROJECT_NAME +PROJECT_NAME=$2 + +## get project full name +setServerName + +COMMAND=$1 +export SERVER_PID=$DSS_PID_DIR/$SERVER_NAME.pid +case $COMMAND in + start|stop|restart|status) + $COMMAND $SERVER_NAME + ;; + *) + print_usage + exit 2 + ;; +esac \ No newline at end of file diff --git a/sbin/dss-start-all.sh b/sbin/dss-start-all.sh new file mode 100644 index 000000000..dea17d632 --- /dev/null +++ b/sbin/dss-start-all.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Start all dss applications +info="We will start all dss applications, it will take some time, please wait" +echo ${info} + +#Actively load user env + + +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +local_host="`hostname --fqdn`" +source $DSS_HOME/sbin/common.sh +source $DSS_CONF_DIR/config.sh + +function startApp(){ +echo "<-------------------------------->" +echo "Begin to start $SERVER_NAME" +SERVER_START_CMD="sh $DSS_INSTALL_HOME/sbin/dss-daemon.sh restart $SERVER_NAME" +if test -z "$SERVER_IP" +then + SERVER_IP=$local_host +fi +executeCMD $SERVER_IP "$SERVER_START_CMD" +echo "End to start $SERVER_NAME" +echo "<-------------------------------->" +sleep 1 +} + +function checkServer() { +echo "<-------------------------------->" +echo "Begin to check $SERVER_NAME" +SERVER_CHECK_CMD="sh $DSS_HOME/sbin/dss-daemon.sh status $SERVER_NAME" +if test -z "$SERVER_IP" +then + SERVER_IP=$local_host +fi + +executeCMD $SERVER_IP "$SERVER_CHECK_CMD" + +if [ $? -ne 0 ]; then + ALL_SERVER_NAME=$SERVER_NAME + LOG_PATH=$DSS_HOME/logs/$ALL_SERVER_NAME.log + echo "ERROR: your $ALL_SERVER_NAME microservice is not start successful !!! ERROR logs as follows :" + echo "Please check detail log, log path :$LOG_PATH" + echo '<---------------------------------------------------->' + executeCMD $ALL_SERVER_NAME "tail -n 50 $LOG_PATH" + echo '<---------------------------------------------------->' + echo "Please check detail log, log path :$LOG_PATH" + exit 1 +fi +echo "<-------------------------------->" +sleep 3 +} + + +function startDssProject(){ + SERVER_NAME=dss-framework-project-server + SERVER_IP=$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP + startApp + sleep 5 + +# echo "------------------------Start to check whether the project service is registered to eureka successfully-----------------------------" +# #project服务启动并注册到eureka后再启动其他服务 +# i=1 +# while [[ -z $result ]] && [[ $i -le 24 ]] +# do +# sleep 5 +# if [ -z $EUREKA_USERNAME ] || [ -z $EUREKA_PASSWORD ];then +# response=`curl http://${EUREKA_INSTALL_IP}:${EUREKA_PORT}/eureka/apps/DSS-FRAMEWORK-PROJECT-SERVER` +# else +# response=`curl http://${EUREKA_USENAME}:${EUREKA_PASSWORD}@${EUREKA_INSTALL_IP}:${EUREKA_PORT}/eureka/apps/DSS-FRAMEWORK-PROJECT-SERVER` +# fi +# let i++ +# result=$(echo $response |grep 'DSS-FRAMEWORK-PROJECT-SERVER') +# done +# if [[ $i -eq 25 ]]; then +# echo "the project server start failed in two minutes,please check the log to find more error details." +# exit +# fi +# echo "------------------------the project service is registered to eureka successfully------------------------------------------------" + + SERVER_NAME=dss-apiservice-server + SERVER_IP=$DSS_APISERVICE_SERVER_INSTALL_IP + startApp + + SERVER_NAME=dss-scriptis-server + SERVER_IP=$DSS_SCRIPTIS_SERVER_INSTALL_IP + startApp + + SERVER_NAME=dss-flow-execution-server + SERVER_IP=$DSS_FLOW_EXECUTION_SERVER_INSTALL_IP + startApp + + SERVER_NAME=dss-data-api-server + SERVER_IP=$DSS_DATA_API_SERVER_INSTALL_IP + startApp + + SERVER_NAME=dss-data-governance-server + SERVER_IP=$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP + startApp + + SERVER_NAME=dss-guide-server + SERVER_IP=$DSS_GUIDE_SERVER_INSTALL_IP + startApp + + SERVER_NAME=dss-framework-orchestrator-server + SERVER_IP=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP + startApp + + SERVER_NAME=dss-workflow-server + SERVER_IP=$DSS_WORKFLOW_SERVER_INSTALL_IP + startApp + +} + +function checkDssService(){ + SERVER_NAME=dss-framework-project-server + SERVER_IP=$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP + checkServer + + SERVER_NAME=dss-framework-orchestrator-server + SERVER_IP=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP + checkServer + + SERVER_NAME=dss-apiservice-server + SERVER_IP=$DSS_APISERVICE_SERVER_INSTALL_IP + checkServer + + SERVER_NAME=dss-scriptis-server + SERVER_IP=$DSS_SCRIPTIS_SERVER_INSTALL_IP + checkServer + + SERVER_NAME=dss-workflow-server + SERVER_IP=$DSS_WORKFLOW_SERVER_INSTALL_IP + checkServer + + SERVER_NAME=dss-flow-execution-server + SERVER_IP=$DSS_FLOW_EXECUTION_SERVER_INSTALL_IP + checkServer + + SERVER_NAME=dss-data-api-server + SERVER_IP=$DSS_DATA_API_SERVER_INSTALL_IP + checkServer + + SERVER_NAME=dss-guide-server + SERVER_IP=$DSS_GUIDE_SERVER_INSTALL_IP + checkServer +} + + + +startDssProject +checkDssService \ No newline at end of file diff --git a/sbin/dss-stop-all.sh b/sbin/dss-stop-all.sh new file mode 100644 index 000000000..55c5c806b --- /dev/null +++ b/sbin/dss-stop-all.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# stop all dss applications +info="We will stop all dss applications, it will take some time, please wait" +echo ${info} + +#Actively load user env + + +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi +local_host="`hostname --fqdn`" +source $DSS_HOME/sbin/common.sh + +function stopApp(){ +echo "<-------------------------------->" +echo "Begin to stop $SERVER_NAME" +SERVER_STOP_CMD="sh $DSS_HOME/sbin/dss-daemon.sh stop $SERVER_NAME" +if test -z "$SERVER_IP" +then + SERVER_IP=$local_host +fi +executeCMD $SERVER_IP "$SERVER_STOP_CMD" +echo "<-------------------------------->" +} + +function stopDssProject(){ + SERVER_NAME=dss-framework-project-server + SERVER_IP=$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP + stopApp + + SERVER_NAME=dss-framework-orchestrator-server + SERVER_IP=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP + stopApp + + SERVER_NAME=dss-apiservice-server + SERVER_IP=$DSS_APISERVICE_SERVER_INSTALL_IP + stopApp + + SERVER_NAME=dss-scriptis-server + SERVER_IP=$DSS_DATAPIPE_SERVER_INSTALL_IP + stopApp + + SERVER_NAME=dss-workflow-server + SERVER_IP=$DSS_WORKFLOW_SERVER_INSTALL_IP + stopApp + + SERVER_NAME=dss-flow-execution-server + SERVER_IP=$DSS_FLOW_EXECUTION_SERVER_INSTALL_IP + stopApp + + SERVER_NAME=dss-data-api-server + SERVER_IP=$DSS_DATA_API_SERVER_INSTALL_IP + stopApp + + SERVER_NAME=dss-data-governance-server + SERVER_IP=$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP + stopApp + + SERVER_NAME=dss-guide-server + SERVER_IP=$DSS_GUIDE_SERVER_INSTALL_IP + stopApp +} + +stopDssProject \ No newline at end of file diff --git a/sbin/ext/dss-apiservice-server b/sbin/ext/dss-apiservice-server new file mode 100644 index 000000000..58ad10ff0 --- /dev/null +++ b/sbin/ext/dss-apiservice-server @@ -0,0 +1,86 @@ +#!/bin/bash +# +# description: ecm start cmd +# + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-apiservice-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.apiservice.DSSApiServiceServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-apps/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-data-api-server b/sbin/ext/dss-data-api-server new file mode 100644 index 000000000..4f9010b9b --- /dev/null +++ b/sbin/ext/dss-data-api-server @@ -0,0 +1,83 @@ +#!/bin/bash + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-data-api-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.data.api.server.DSSDataApiApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-data-api/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-data-governance-server b/sbin/ext/dss-data-governance-server new file mode 100644 index 000000000..8b259fe74 --- /dev/null +++ b/sbin/ext/dss-data-governance-server @@ -0,0 +1,85 @@ +#!/bin/bash +# +# description: ecm start cmd +# +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-data-governance-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.data.governance.DSSDataGovernanceApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-data-governance/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-flow-execution-server b/sbin/ext/dss-flow-execution-server new file mode 100644 index 000000000..4b652c945 --- /dev/null +++ b/sbin/ext/dss-flow-execution-server @@ -0,0 +1,83 @@ +#!/bin/bash + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-flow-execution-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.flow.execution.entrance.DSSFowExecutionServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-orchestrator/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-framework-orchestrator-server b/sbin/ext/dss-framework-orchestrator-server new file mode 100644 index 000000000..a1af76c97 --- /dev/null +++ b/sbin/ext/dss-framework-orchestrator-server @@ -0,0 +1,85 @@ +#!/bin/bash +# +# description: ecm start cmd + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-framework-orchestrator-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.orchestrator.server.DSSOrchestratorServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-framework/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-framework-project-server b/sbin/ext/dss-framework-project-server new file mode 100644 index 000000000..ac969bb67 --- /dev/null +++ b/sbin/ext/dss-framework-project-server @@ -0,0 +1,83 @@ +#!/bin/bash + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-framework-project-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.framework.project.server.DSSProjectServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-framework/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-guide-server b/sbin/ext/dss-guide-server new file mode 100644 index 000000000..a6eb96808 --- /dev/null +++ b/sbin/ext/dss-guide-server @@ -0,0 +1,85 @@ +#!/bin/bash +# +# description: ecm start cmd +# +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-guide-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.guide.server.DSSGuideApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-guide/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-scriptis-server b/sbin/ext/dss-scriptis-server new file mode 100644 index 000000000..ac4e47924 --- /dev/null +++ b/sbin/ext/dss-scriptis-server @@ -0,0 +1,83 @@ +#!/bin/bash + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-scriptis-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.scriptis.DSSScriptisServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-apps/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-workflow-server b/sbin/ext/dss-workflow-server new file mode 100644 index 000000000..12b3c0be9 --- /dev/null +++ b/sbin/ext/dss-workflow-server @@ -0,0 +1,86 @@ +#!/bin/bash +# +# description: ecm start cmd +# + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-workflow-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.workflow.DSSWorkflowServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-orchestrator/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/k8s/dss-apiservice-server.sh b/sbin/k8s/dss-apiservice-server.sh new file mode 100644 index 000000000..9629b2401 --- /dev/null +++ b/sbin/k8s/dss-apiservice-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-apiservice-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx512M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.apiservice.DSSApiServiceServerApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS \ No newline at end of file diff --git a/sbin/k8s/dss-data-api-server.sh b/sbin/k8s/dss-data-api-server.sh new file mode 100644 index 000000000..ef6e44083 --- /dev/null +++ b/sbin/k8s/dss-data-api-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-data-api-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx2048M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.data.api.server.DSSDataApiApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/sbin/k8s/dss-data-governance-server.sh b/sbin/k8s/dss-data-governance-server.sh new file mode 100644 index 000000000..76aca5370 --- /dev/null +++ b/sbin/k8s/dss-data-governance-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-data-governance-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx2048M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.data.governance.DSSDataGovernanceApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/sbin/k8s/dss-flow-execution-server.sh b/sbin/k8s/dss-flow-execution-server.sh new file mode 100644 index 000000000..f6d70127f --- /dev/null +++ b/sbin/k8s/dss-flow-execution-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-flow-execution-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx512M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.flow.execution.entrance.DSSFowExecutionServerApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS \ No newline at end of file diff --git a/sbin/k8s/dss-framework-orchestrator-server.sh b/sbin/k8s/dss-framework-orchestrator-server.sh new file mode 100644 index 000000000..a4fa1351e --- /dev/null +++ b/sbin/k8s/dss-framework-orchestrator-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-framework-orchestrator-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx512M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.orchestrator.server.DSSOrchestratorServerApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS \ No newline at end of file diff --git a/sbin/k8s/dss-framework-project-server.sh b/sbin/k8s/dss-framework-project-server.sh new file mode 100644 index 000000000..f7dea9f31 --- /dev/null +++ b/sbin/k8s/dss-framework-project-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-framework-project-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx512M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.framework.project.server.DSSProjectServerApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/sbin/k8s/dss-guide-server.sh b/sbin/k8s/dss-guide-server.sh new file mode 100644 index 000000000..3bbff2b77 --- /dev/null +++ b/sbin/k8s/dss-guide-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-guide-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx2048M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.guide.server.DSSGuideApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/sbin/k8s/dss-workflow-server.sh b/sbin/k8s/dss-workflow-server.sh new file mode 100644 index 000000000..ae9a62715 --- /dev/null +++ b/sbin/k8s/dss-workflow-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-workflow-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx512M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.workflow.DSSWorkflowServerApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/web/.editorconfig b/web/.editorconfig new file mode 100644 index 000000000..f1cc3ad32 --- /dev/null +++ b/web/.editorconfig @@ -0,0 +1,15 @@ +# http://editorconfig.org + +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +insert_final_newline = false +trim_trailing_whitespace = false diff --git a/web/.eslintignore b/web/.eslintignore new file mode 100644 index 000000000..d1a501255 --- /dev/null +++ b/web/.eslintignore @@ -0,0 +1,4 @@ +iconfont.js +packages/dss/server/* +packages/dss/dist/*/* +"*.min.js" diff --git a/web/.eslintrc.js b/web/.eslintrc.js new file mode 100644 index 000000000..fe93fbe50 --- /dev/null +++ b/web/.eslintrc.js @@ -0,0 +1,93 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +module.exports = { + root: true, + env: { + node: true + }, + extends: [ + 'eslint:recommended', + 'plugin:vue/essential' + ], + rules: { + 'no-console': process.env.NODE_ENV === 'production' ? 'error' : 'off', + 'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off', + 'key-spacing': ['error'], + 'standard/no-callback-literal': 0, + 'handle-callback-err': 0, + 'no-return-assign': 0, + 'eqeqeq': 0, + 'comma-dangle': 0, + 'semi': 0, + 'space-before-function-paren': 0, + 'keyword-spacing': 0, + 'no-useless-escape': 0, + 'operator-linebreak': 0, + 'indent': [ + 'error', + 2, + { + 'SwitchCase': 1 + } + ], + 'no-const-assign': 'warn', + 'no-this-before-super': 'warn', + "no-irregular-whitespace": 0, + 'no-undef': 2, + 'no-unreachable': 'warn', + 'no-unused-vars': 2, + 'constructor-super': 'warn', + 'valid-typeof': 'warn', + 'one-var': 'warn', + 'max-len': 'off', + 'no-trailing-spaces': 'off', + 'require-jsdoc': 'warn', + 'camelcase': 'warn', + 'no-invalid-this': 'off', + 'linebreak-style': 0, + 'vue/no-parsing-error': [2, { + 'x-invalid-end-tag': false, + 'invalid-first-character-of-tag-name': false + }], + 'no-tabs': 0, + 'vue/html-indent': [2, 2, { + 'attribute': 1, + 'closeBracket': 0, + 'alignAttributesVertically': false + }], + 'vue/require-default-prop': 0, + 'vue/component-name-in-template-casing': 0, + 'vue/html-closing-bracket-spacing': 0, + 'vue/html-closing-bracket-newline': 0, + 'vue/singleline-html-element-content-newline': 0, + 'vue/multiline-html-element-content-newline': 0, + 'vue/attributes-order': 0, + 'vue/html-self-closing': 0, + 'no-useless-constructor': 0, + 'no-mixed-operators': 0, + 'no-new-func': 0, + 'no-template-curly-in-string': 0, + 'no-useless-call': 0, + "one-var": 0, + "camelcase": 0 + }, + parserOptions: { + "parser": 'babel-eslint', + "sourceType": "module" + } +} diff --git a/web/.gitattributes b/web/.gitattributes new file mode 100644 index 000000000..ec8935df7 --- /dev/null +++ b/web/.gitattributes @@ -0,0 +1,9 @@ +* text=auto +* text eol=lf +*.png binary +*.gif binary +*.ttf binary +*.woff binary +*.eot binary +*.woff binary +*.otf binary \ No newline at end of file diff --git a/web/.gitignore b/web/.gitignore new file mode 100644 index 000000000..6628e60c0 --- /dev/null +++ b/web/.gitignore @@ -0,0 +1,14 @@ +.DS_Store +.vscode +.cache +.idea/ + +node_modules/ +dist/ + +package-lock.json +yarn-eror.log +wedatasphere-*.zip +luban-*.zip +dist.zip +.env.* diff --git a/web/.jshintrc b/web/.jshintrc new file mode 100644 index 000000000..40bd84d8e --- /dev/null +++ b/web/.jshintrc @@ -0,0 +1,28 @@ +{ + "node": true, + "esnext": true, + "bitwise": true, + "camelcase": true, + "curly": true, + "eqeqeq": true, + "immed": true, + "indent": 2, + "latedef": true, + "newcap": true, + "noarg": true, + "quotmark": "single", + "regexp": true, + "undef": true, + "unused": false, + "strict": true, + "trailing": true, + "smarttabs": true, + "white": true, + "globals": { + "history": false, + "window": false, + "_": false, + "cordova": false, + "angular": false + } +} diff --git a/web/.prettierrc.json b/web/.prettierrc.json new file mode 100644 index 000000000..f3b463e7a --- /dev/null +++ b/web/.prettierrc.json @@ -0,0 +1,4 @@ +{ + "singleQuote": true, + "semi": false +} \ No newline at end of file diff --git a/web/Dockerfile b/web/Dockerfile new file mode 100644 index 000000000..88d9b73a8 --- /dev/null +++ b/web/Dockerfile @@ -0,0 +1,6 @@ +FROM nm.hub.com/library/nginx + +WORKDIR /usr/share/nginx/html + +COPY web/dist/dist /usr/share/nginx/html/dss +COPY linkisweb/web/dist/dist /usr/share/nginx/html/linkis diff --git a/web/LICENSE b/web/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/web/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/web/README-DEV.md b/web/README-DEV.md new file mode 100644 index 000000000..75d4c5b09 --- /dev/null +++ b/web/README-DEV.md @@ -0,0 +1,68 @@ +### 1.1.0版本说明 + +- 前端工程管理方面达到目录结构清晰,方便维护及迭代开发 +- 可选模块构建,产出不同用户的应用 +- 插件开发支持 + +### 项目结构 + +``` +├─dist # 构建后静态资源 +├─docs # 文档 +├─node_modules +└─packages # 各应用模块 + ├─apiServices + ├─dataGovernance + ├─dataService + ├─dolphinScheduler + ├─dss + ├─scheduleCenter + ├─scriptis + ├─shared + │ ├─common + │ └─components + ├─workflows + └─workspace + +``` + +### 建议/约束 + +新增功能模块先确定涉及应用,按照上面目录结构维护代码同时建议遵守以下约束: + +- 子应用可以配置自己的layout需要在应用router模块导出配置subAppRoutes +- 子应用支持使用自己的header,需要在config.json里配置模块路径 +- 各应用需要使用iView作为UI库,并提供路由,国际化等配置写入config.json +- 各应用间不要相互直接依赖,确有依赖通过lerna管理 +- 可复用组件,资源需要合理放置,packages/shared 共享组件方法,修改需要注意影响 +- 各应用路由应以应用名做统一前缀 +- 各应用之间需要事件通信,应当在config.json 里声明对应module文件路径 +- 新增功能模块需要按照现有目录约束建立文件,已有功能修改应在有限模块内进行,控制影响范围 +- 全局共用组件、公共基础样式、工具方法修改需评估后才能修改,并且重点review +- 插件扩展开发,扩展点增加,修改需要讨论,注意兼容 + +### 如何新增一个子应用,如何扩展 + +1. config.json 新增应用配置 +2. packages 下新建应用目录或者插件目录进行应用开发 + + +### 前端开发、构建打包 + +``` +# 安装依赖 +lerna bootstrap +# 添加依赖通过lerna add 添加 +lerna add [@version] +# 指定组件添加依赖 +lerna add [@version] --@scope=@dataspherestudio/workspace +# 开发启动DSS +npm run serve +# 运行部分模块子应用,支持通过module组合。如科管版本: +npm run serve --module=scriptis +# 打包DSS应用 +npm run build +# 打包子应用,支持通过module组合 +npm run build --module=scriptis +npm run build --module=apiServices,workspace --micro_module=apiServices +``` diff --git a/web/config.json b/web/config.json new file mode 100644 index 000000000..050e03d02 --- /dev/null +++ b/web/config.json @@ -0,0 +1,83 @@ +{ + "apps": { + "workspace": { + "routes": "workspace/router", + "module": "workspace/module", + "i18n": { + "en": "workspace/i18n/en.json", + "zh-CN": "workspace/i18n/zh.json" + } + }, + "scriptis": { + "routes": "scriptis/router", + "module": "scriptis/module", + "i18n": { + "en": "scriptis/i18n/common/en.json", + "zh-CN": "scriptis/i18n/common/zh.json" + } + }, + "workflows": { + "routes": "workflows/router", + "module": "workflows/module", + "i18n": { + "en": "workflows/i18n/common/en.json", + "zh-CN": "workflows/i18n/common/zh.json" + } + }, + "dataService": { + "routes": "dataService/router", + "module": "dataService/module", + "i18n": { + "en": "dataService/i18n/en.json", + "zh-CN": "dataService/i18n/zh.json" + } + }, + "apiServices": { + "routes": "apiServices/router", + "module": "apiServices/module", + "i18n": { + "en": "apiServices/i18n/en.json", + "zh-CN": "apiServices/i18n/zh.json" + } + }, + "dataGovernance": { + "routes": "dataGovernance/router", + "module": "dataGovernance/module", + "i18n": { + "en": "dataGovernance/i18n/en.json", + "zh-CN": "dataGovernance/i18n/zh.json" + } + }, + "dolphinScheduler": { + "routes": "dolphinScheduler/router", + "module": "dolphinScheduler/module", + "i18n": { + "en": "dolphinScheduler/i18n/en.json", + "zh-CN": "dolphinScheduler/i18n/zh.json" + } + } + }, + "exts": { + "dss-plugin-open": { + "module": "exts/open-source/index.js", + "i18n": { + "en": "exts/open-source/i18n/en.json", + "zh-CN": "exts/open-source/i18n/zh.json" + }, + "options": null + } + }, + "conf": { + "app_name": "DataSphere Studio", + "app_logo": "dss/assets/images/dssLogo.png", + "user_guide": "", + "hide_view_tb_detail": true, + "hide_view_db_detail": true, + "watermark": { + "show": false, + "template": "${username} ${time}", + "timeupdate": 60000 + } + }, + "version": "1.1.1" +} diff --git a/web/lerna.json b/web/lerna.json new file mode 100644 index 000000000..62c26edb5 --- /dev/null +++ b/web/lerna.json @@ -0,0 +1,12 @@ +{ + "packages": [ + "packages/*" + ], + "command": { + "bootstrap": { + "hoist": true + } + }, + "npmClient": "npm", + "version": "1.1.4" +} diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 000000000..179553126 --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,22289 @@ +{ + "name": "dataspherestudio", + "version": "1.1.4", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@ampproject/remapping": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/@ampproject/remapping/-/remapping-2.1.2.tgz", + "integrity": "sha512-hoyByceqwKirw7w3Z7gnIIZC3Wx3J484Y3L/cMpXFbr7d9ZQj2mODrirNzcJa+SM3UlpWXYvKV4RlRpFXlWgXg==", + "dev": true, + "requires": { + "@jridgewell/trace-mapping": "^0.3.0" + } + }, + "@antv/hierarchy": { + "version": "0.6.8", + "resolved": "https://registry.npmmirror.com/@antv/hierarchy/-/hierarchy-0.6.8.tgz", + "integrity": "sha512-wVzUl+pxny5gyGJ2mkWx8IiEypX6bnMHgr/NILgbxY6shoy0Vf4FhZpI3CY8Ez7bQT6js8fMkB2NymPW7d7i8A==", + "requires": { + "@antv/util": "^2.0.7" + } + }, + "@antv/matrix-util": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/@antv/matrix-util/-/matrix-util-3.0.4.tgz", + "integrity": "sha512-BAPyu6dUliHcQ7fm9hZSGKqkwcjEDVLVAstlHULLvcMZvANHeLXgHEgV7JqcAV/GIhIz8aZChIlzM1ZboiXpYQ==", + "requires": { + "@antv/util": "^2.0.9", + "gl-matrix": "^3.3.0", + "tslib": "^2.0.3" + } + }, + "@antv/util": { + "version": "2.0.17", + "resolved": "https://registry.npmmirror.com/@antv/util/-/util-2.0.17.tgz", + "integrity": "sha512-o6I9hi5CIUvLGDhth0RxNSFDRwXeywmt6ExR4+RmVAzIi48ps6HUy+svxOCayvrPBN37uE6TAc2KDofRo0nK9Q==", + "requires": { + "csstype": "^3.0.8", + "tslib": "^2.0.3" + } + }, + "@babel/code-frame": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/code-frame/-/code-frame-7.16.7.tgz", + "integrity": "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==", + "dev": true, + "requires": { + "@babel/highlight": "^7.16.7" + } + }, + "@babel/compat-data": { + "version": "7.17.7", + "resolved": "https://registry.npmmirror.com/@babel/compat-data/-/compat-data-7.17.7.tgz", + "integrity": "sha512-p8pdE6j0a29TNGebNm7NzYZWB3xVZJBZ7XGs42uAKzQo8VQ3F0By/cQCtUEABwIqw5zo6WA4NbmxsfzADzMKnQ==", + "dev": true + }, + "@babel/core": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/core/-/core-7.17.9.tgz", + "integrity": "sha512-5ug+SfZCpDAkVp9SFIZAzlW18rlzsOcJGaetCjkySnrXXDUw9AR8cDUm1iByTmdWM6yxX6/zycaV76w3YTF2gw==", + "dev": true, + "requires": { + "@ampproject/remapping": "^2.1.0", + "@babel/code-frame": "^7.16.7", + "@babel/generator": "^7.17.9", + "@babel/helper-compilation-targets": "^7.17.7", + "@babel/helper-module-transforms": "^7.17.7", + "@babel/helpers": "^7.17.9", + "@babel/parser": "^7.17.9", + "@babel/template": "^7.16.7", + "@babel/traverse": "^7.17.9", + "@babel/types": "^7.17.0", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.1", + "semver": "^6.3.0" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "@babel/generator": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/generator/-/generator-7.17.9.tgz", + "integrity": "sha512-rAdDousTwxbIxbz5I7GEQ3lUip+xVCXooZNbsydCWs3xA7ZsYOv+CFRdzGxRX78BmQHu9B1Eso59AOZQOJDEdQ==", + "dev": true, + "requires": { + "@babel/types": "^7.17.0", + "jsesc": "^2.5.1", + "source-map": "^0.5.0" + } + }, + "@babel/helper-annotate-as-pure": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.16.7.tgz", + "integrity": "sha512-s6t2w/IPQVTAET1HitoowRGXooX8mCgtuP5195wD/QJPV6wYjpujCGF7JuMODVX2ZAJOf1GT6DT9MHEZvLOFSw==", + "dev": true, + "requires": { + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.16.7.tgz", + "integrity": "sha512-C6FdbRaxYjwVu/geKW4ZeQ0Q31AftgRcdSnZ5/jsH6BzCJbtvXvhpfkbkThYSuutZA7nCXpPR6AD9zd1dprMkA==", + "dev": true, + "requires": { + "@babel/helper-explode-assignable-expression": "^7.16.7", + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-compilation-targets": { + "version": "7.17.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.17.7.tgz", + "integrity": "sha512-UFzlz2jjd8kroj0hmCFV5zr+tQPi1dpC2cRsDV/3IEW8bJfCPrPpmcSN6ZS8RqIq4LXcmpipCQFPddyFA5Yc7w==", + "dev": true, + "requires": { + "@babel/compat-data": "^7.17.7", + "@babel/helper-validator-option": "^7.16.7", + "browserslist": "^4.17.5", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "@babel/helper-create-class-features-plugin": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.17.9.tgz", + "integrity": "sha512-kUjip3gruz6AJKOq5i3nC6CoCEEF/oHH3cp6tOZhB+IyyyPyW0g1Gfsxn3mkk6S08pIA2y8GQh609v9G/5sHVQ==", + "dev": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.16.7", + "@babel/helper-environment-visitor": "^7.16.7", + "@babel/helper-function-name": "^7.17.9", + "@babel/helper-member-expression-to-functions": "^7.17.7", + "@babel/helper-optimise-call-expression": "^7.16.7", + "@babel/helper-replace-supers": "^7.16.7", + "@babel/helper-split-export-declaration": "^7.16.7" + } + }, + "@babel/helper-create-regexp-features-plugin": { + "version": "7.17.0", + "resolved": "https://registry.npmmirror.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.17.0.tgz", + "integrity": "sha512-awO2So99wG6KnlE+TPs6rn83gCz5WlEePJDTnLEqbchMVrBeAujURVphRdigsk094VhvZehFoNOihSlcBjwsXA==", + "dev": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.16.7", + "regexpu-core": "^5.0.1" + } + }, + "@babel/helper-define-polyfill-provider": { + "version": "0.3.1", + "resolved": "https://registry.npmmirror.com/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.1.tgz", + "integrity": "sha512-J9hGMpJQmtWmj46B3kBHmL38UhJGhYX7eqkcq+2gsstyYt341HmPeWspihX43yVRA0mS+8GGk2Gckc7bY/HCmA==", + "dev": true, + "requires": { + "@babel/helper-compilation-targets": "^7.13.0", + "@babel/helper-module-imports": "^7.12.13", + "@babel/helper-plugin-utils": "^7.13.0", + "@babel/traverse": "^7.13.0", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2", + "semver": "^6.1.2" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "@babel/helper-environment-visitor": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.16.7.tgz", + "integrity": "sha512-SLLb0AAn6PkUeAfKJCCOl9e1R53pQlGAfc4y4XuMRZfqeMYLE0dM1LMhqbGAlGQY0lfw5/ohoYWAe9V1yibRag==", + "dev": true, + "requires": { + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-explode-assignable-expression": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.16.7.tgz", + "integrity": "sha512-KyUenhWMC8VrxzkGP0Jizjo4/Zx+1nNZhgocs+gLzyZyB8SHidhoq9KK/8Ato4anhwsivfkBLftky7gvzbZMtQ==", + "dev": true, + "requires": { + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-function-name": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/helper-function-name/-/helper-function-name-7.17.9.tgz", + "integrity": "sha512-7cRisGlVtiVqZ0MW0/yFB4atgpGLWEHUVYnb448hZK4x+vih0YO5UoS11XIYtZYqHd0dIPMdUSv8q5K4LdMnIg==", + "dev": true, + "requires": { + "@babel/template": "^7.16.7", + "@babel/types": "^7.17.0" + } + }, + "@babel/helper-hoist-variables": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.16.7.tgz", + "integrity": "sha512-m04d/0Op34H5v7pbZw6pSKP7weA6lsMvfiIAMeIvkY/R4xQtBSMFEigu9QTZ2qB/9l22vsxtM8a+Q8CzD255fg==", + "dev": true, + "requires": { + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-member-expression-to-functions": { + "version": "7.17.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.17.7.tgz", + "integrity": "sha512-thxXgnQ8qQ11W2wVUObIqDL4p148VMxkt5T/qpN5k2fboRyzFGFmKsTGViquyM5QHKUy48OZoca8kw4ajaDPyw==", + "dev": true, + "requires": { + "@babel/types": "^7.17.0" + } + }, + "@babel/helper-module-imports": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-module-imports/-/helper-module-imports-7.16.7.tgz", + "integrity": "sha512-LVtS6TqjJHFc+nYeITRo6VLXve70xmq7wPhWTqDJusJEgGmkAACWwMiTNrvfoQo6hEhFwAIixNkvB0jPXDL8Wg==", + "dev": true, + "requires": { + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-module-transforms": { + "version": "7.17.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-module-transforms/-/helper-module-transforms-7.17.7.tgz", + "integrity": "sha512-VmZD99F3gNTYB7fJRDTi+u6l/zxY0BE6OIxPSU7a50s6ZUQkHwSDmV92FfM+oCG0pZRVojGYhkR8I0OGeCVREw==", + "dev": true, + "requires": { + "@babel/helper-environment-visitor": "^7.16.7", + "@babel/helper-module-imports": "^7.16.7", + "@babel/helper-simple-access": "^7.17.7", + "@babel/helper-split-export-declaration": "^7.16.7", + "@babel/helper-validator-identifier": "^7.16.7", + "@babel/template": "^7.16.7", + "@babel/traverse": "^7.17.3", + "@babel/types": "^7.17.0" + } + }, + "@babel/helper-optimise-call-expression": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.16.7.tgz", + "integrity": "sha512-EtgBhg7rd/JcnpZFXpBy0ze1YRfdm7BnBX4uKMBd3ixa3RGAE002JZB66FJyNH7g0F38U05pXmA5P8cBh7z+1w==", + "dev": true, + "requires": { + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-plugin-utils": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.16.7.tgz", + "integrity": "sha512-Qg3Nk7ZxpgMrsox6HreY1ZNKdBq7K72tDSliA6dCl5f007jR4ne8iD5UzuNnCJH2xBf2BEEVGr+/OL6Gdp7RxA==", + "dev": true + }, + "@babel/helper-remap-async-to-generator": { + "version": "7.16.8", + "resolved": "https://registry.npmmirror.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.16.8.tgz", + "integrity": "sha512-fm0gH7Flb8H51LqJHy3HJ3wnE1+qtYR2A99K06ahwrawLdOFsCEWjZOrYricXJHoPSudNKxrMBUPEIPxiIIvBw==", + "dev": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.16.7", + "@babel/helper-wrap-function": "^7.16.8", + "@babel/types": "^7.16.8" + } + }, + "@babel/helper-replace-supers": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-replace-supers/-/helper-replace-supers-7.16.7.tgz", + "integrity": "sha512-y9vsWilTNaVnVh6xiJfABzsNpgDPKev9HnAgz6Gb1p6UUwf9NepdlsV7VXGCftJM+jqD5f7JIEubcpLjZj5dBw==", + "dev": true, + "requires": { + "@babel/helper-environment-visitor": "^7.16.7", + "@babel/helper-member-expression-to-functions": "^7.16.7", + "@babel/helper-optimise-call-expression": "^7.16.7", + "@babel/traverse": "^7.16.7", + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-simple-access": { + "version": "7.17.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-simple-access/-/helper-simple-access-7.17.7.tgz", + "integrity": "sha512-txyMCGroZ96i+Pxr3Je3lzEJjqwaRC9buMUgtomcrLe5Nd0+fk1h0LLA+ixUF5OW7AhHuQ7Es1WcQJZmZsz2XA==", + "dev": true, + "requires": { + "@babel/types": "^7.17.0" + } + }, + "@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.16.0", + "resolved": "https://registry.npmmirror.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.16.0.tgz", + "integrity": "sha512-+il1gTy0oHwUsBQZyJvukbB4vPMdcYBrFHa0Uc4AizLxbq6BOYC51Rv4tWocX9BLBDLZ4kc6qUFpQ6HRgL+3zw==", + "dev": true, + "requires": { + "@babel/types": "^7.16.0" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.16.7.tgz", + "integrity": "sha512-xbWoy/PFoxSWazIToT9Sif+jJTlrMcndIsaOKvTA6u7QEo7ilkRZpjew18/W3c7nm8fXdUDXh02VXTbZ0pGDNw==", + "dev": true, + "requires": { + "@babel/types": "^7.16.7" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz", + "integrity": "sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw==", + "dev": true + }, + "@babel/helper-validator-option": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-validator-option/-/helper-validator-option-7.16.7.tgz", + "integrity": "sha512-TRtenOuRUVo9oIQGPC5G9DgK4743cdxvtOw0weQNpZXaS16SCBi5MNjZF8vba3ETURjZpTbVn7Vvcf2eAwFozQ==", + "dev": true + }, + "@babel/helper-wrap-function": { + "version": "7.16.8", + "resolved": "https://registry.npmmirror.com/@babel/helper-wrap-function/-/helper-wrap-function-7.16.8.tgz", + "integrity": "sha512-8RpyRVIAW1RcDDGTA+GpPAwV22wXCfKOoM9bet6TLkGIFTkRQSkH1nMQ5Yet4MpoXe1ZwHPVtNasc2w0uZMqnw==", + "dev": true, + "requires": { + "@babel/helper-function-name": "^7.16.7", + "@babel/template": "^7.16.7", + "@babel/traverse": "^7.16.8", + "@babel/types": "^7.16.8" + } + }, + "@babel/helpers": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/helpers/-/helpers-7.17.9.tgz", + "integrity": "sha512-cPCt915ShDWUEzEp3+UNRktO2n6v49l5RSnG9M5pS24hA+2FAc5si+Pn1i4VVbQQ+jh+bIZhPFQOJOzbrOYY1Q==", + "dev": true, + "requires": { + "@babel/template": "^7.16.7", + "@babel/traverse": "^7.17.9", + "@babel/types": "^7.17.0" + } + }, + "@babel/highlight": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/highlight/-/highlight-7.17.9.tgz", + "integrity": "sha512-J9PfEKCbFIv2X5bjTMiZu6Vf341N05QIY+d6FvVKynkG1S7G0j3I0QoRtWIrXhZ+/Nlb5Q0MzqL7TokEJ5BNHg==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.16.7", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, + "@babel/parser": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/parser/-/parser-7.17.9.tgz", + "integrity": "sha512-vqUSBLP8dQHFPdPi9bc5GK9vRkYHJ49fsZdtoJ8EQ8ibpwk5rPKfvNIwChB0KVXcIjcepEBBd2VHC5r9Gy8ueg==", + "dev": true + }, + "@babel/plugin-proposal-async-generator-functions": { + "version": "7.16.8", + "resolved": "https://registry.npmmirror.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.16.8.tgz", + "integrity": "sha512-71YHIvMuiuqWJQkebWJtdhQTfd4Q4mF76q2IX37uZPkG9+olBxsX+rH1vkhFto4UeJZ9dPY2s+mDvhDm1u2BGQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/helper-remap-async-to-generator": "^7.16.8", + "@babel/plugin-syntax-async-generators": "^7.8.4" + } + }, + "@babel/plugin-proposal-class-properties": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.16.7.tgz", + "integrity": "sha512-IobU0Xme31ewjYOShSIqd/ZGM/r/cuOz2z0MDbNrhF5FW+ZVgi0f2lyeoj9KFPDOAqsYxmLWZte1WOwlvY9aww==", + "dev": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-proposal-decorators": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.17.9.tgz", + "integrity": "sha512-EfH2LZ/vPa2wuPwJ26j+kYRkaubf89UlwxKXtxqEm57HrgSEYDB8t4swFP+p8LcI9yiP9ZRJJjo/58hS6BnaDA==", + "dev": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.17.9", + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/helper-replace-supers": "^7.16.7", + "@babel/helper-split-export-declaration": "^7.16.7", + "@babel/plugin-syntax-decorators": "^7.17.0", + "charcodes": "^0.2.0" + } + }, + "@babel/plugin-proposal-json-strings": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.16.7.tgz", + "integrity": "sha512-lNZ3EEggsGY78JavgbHsK9u5P3pQaW7k4axlgFLYkMd7UBsiNahCITShLjNQschPyjtO6dADrL24757IdhBrsQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/plugin-syntax-json-strings": "^7.8.3" + } + }, + "@babel/plugin-proposal-object-rest-spread": { + "version": "7.17.3", + "resolved": "https://registry.npmmirror.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.17.3.tgz", + "integrity": "sha512-yuL5iQA/TbZn+RGAfxQXfi7CNLmKi1f8zInn4IgobuCWcAb7i+zj4TYzQ9l8cEzVyJ89PDGuqxK1xZpUDISesw==", + "dev": true, + "requires": { + "@babel/compat-data": "^7.17.0", + "@babel/helper-compilation-targets": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.16.7" + } + }, + "@babel/plugin-proposal-optional-catch-binding": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.16.7.tgz", + "integrity": "sha512-eMOH/L4OvWSZAE1VkHbr1vckLG1WUcHGJSLqqQwl2GaUqG6QjddvrOaTUMNYiv77H5IKPMZ9U9P7EaHwvAShfA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3" + } + }, + "@babel/plugin-proposal-unicode-property-regex": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.16.7.tgz", + "integrity": "sha512-QRK0YI/40VLhNVGIjRNAAQkEHws0cswSdFFjpFyt943YmJIU1da9uW63Iu6NFV6CxTZW5eTDCrwZUstBWgp/Rg==", + "dev": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmmirror.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-decorators": { + "version": "7.17.0", + "resolved": "https://registry.npmmirror.com/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.17.0.tgz", + "integrity": "sha512-qWe85yCXsvDEluNP0OyeQjH63DlhAR3W7K9BxxU1MvbDb48tgBG+Ao6IJJ6smPDrrVzSQZrbF6donpkFBMcs3A==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmmirror.com/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmmirror.com/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-jsx": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.16.7.tgz", + "integrity": "sha512-Esxmk7YjA8QysKeT3VhTXvF6y77f/a91SIs4pWb4H2eWGQkCKFgQaG6hdoEVZtGsrAcb2K5BW66XsOErD4WU3Q==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmmirror.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmmirror.com/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-transform-arrow-functions": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.16.7.tgz", + "integrity": "sha512-9ffkFFMbvzTvv+7dTp/66xvZAWASuPD5Tl9LK3Z9vhOmANo6j94rik+5YMBt4CwHVMWLWpMsriIc2zsa3WW3xQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-async-to-generator": { + "version": "7.16.8", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.16.8.tgz", + "integrity": "sha512-MtmUmTJQHCnyJVrScNzNlofQJ3dLFuobYn3mwOTKHnSCMtbNsqvF71GQmJfFjdrXSsAA7iysFmYWw4bXZ20hOg==", + "dev": true, + "requires": { + "@babel/helper-module-imports": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/helper-remap-async-to-generator": "^7.16.8" + } + }, + "@babel/plugin-transform-block-scoped-functions": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.16.7.tgz", + "integrity": "sha512-JUuzlzmF40Z9cXyytcbZEZKckgrQzChbQJw/5PuEHYeqzCsvebDx0K0jWnIIVcmmDOAVctCgnYs0pMcrYj2zJg==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-block-scoping": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.16.7.tgz", + "integrity": "sha512-ObZev2nxVAYA4bhyusELdo9hb3H+A56bxH3FZMbEImZFiEDYVHXQSJ1hQKFlDnlt8G9bBrCZ5ZpURZUrV4G5qQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-classes": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.16.7.tgz", + "integrity": "sha512-WY7og38SFAGYRe64BrjKf8OrE6ulEHtr5jEYaZMwox9KebgqPi67Zqz8K53EKk1fFEJgm96r32rkKZ3qA2nCWQ==", + "dev": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.16.7", + "@babel/helper-environment-visitor": "^7.16.7", + "@babel/helper-function-name": "^7.16.7", + "@babel/helper-optimise-call-expression": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/helper-replace-supers": "^7.16.7", + "@babel/helper-split-export-declaration": "^7.16.7", + "globals": "^11.1.0" + } + }, + "@babel/plugin-transform-computed-properties": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.16.7.tgz", + "integrity": "sha512-gN72G9bcmenVILj//sv1zLNaPyYcOzUho2lIJBMh/iakJ9ygCo/hEF9cpGb61SCMEDxbbyBoVQxrt+bWKu5KGw==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-destructuring": { + "version": "7.17.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.17.7.tgz", + "integrity": "sha512-XVh0r5yq9sLR4vZ6eVZe8FKfIcSgaTBxVBRSYokRj2qksf6QerYnTxz9/GTuKTH/n/HwLP7t6gtlybHetJ/6hQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-dotall-regex": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.16.7.tgz", + "integrity": "sha512-Lyttaao2SjZF6Pf4vk1dVKv8YypMpomAbygW+mU5cYP3S5cWTfCJjG8xV6CFdzGFlfWK81IjL9viiTvpb6G7gQ==", + "dev": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-duplicate-keys": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.16.7.tgz", + "integrity": "sha512-03DvpbRfvWIXyK0/6QiR1KMTWeT6OcQ7tbhjrXyFS02kjuX/mu5Bvnh5SDSWHxyawit2g5aWhKwI86EE7GUnTw==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-exponentiation-operator": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.16.7.tgz", + "integrity": "sha512-8UYLSlyLgRixQvlYH3J2ekXFHDFLQutdy7FfFAMm3CPZ6q9wHCwnUyiXpQCe3gVVnQlHc5nsuiEVziteRNTXEA==", + "dev": true, + "requires": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-for-of": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.16.7.tgz", + "integrity": "sha512-/QZm9W92Ptpw7sjI9Nx1mbcsWz33+l8kuMIQnDwgQBG5s3fAfQvkRjQ7NqXhtNcKOnPkdICmUHyCaWW06HCsqg==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-function-name": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.16.7.tgz", + "integrity": "sha512-SU/C68YVwTRxqWj5kgsbKINakGag0KTgq9f2iZEXdStoAbOzLHEBRYzImmA6yFo8YZhJVflvXmIHUO7GWHmxxA==", + "dev": true, + "requires": { + "@babel/helper-compilation-targets": "^7.16.7", + "@babel/helper-function-name": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-literals": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.16.7.tgz", + "integrity": "sha512-6tH8RTpTWI0s2sV6uq3e/C9wPo4PTqqZps4uF0kzQ9/xPLFQtipynvmT1g/dOfEJ+0EQsHhkQ/zyRId8J2b8zQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-modules-amd": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.16.7.tgz", + "integrity": "sha512-KaaEtgBL7FKYwjJ/teH63oAmE3lP34N3kshz8mm4VMAw7U3PxjVwwUmxEFksbgsNUaO3wId9R2AVQYSEGRa2+g==", + "dev": true, + "requires": { + "@babel/helper-module-transforms": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-commonjs": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.17.9.tgz", + "integrity": "sha512-2TBFd/r2I6VlYn0YRTz2JdazS+FoUuQ2rIFHoAxtyP/0G3D82SBLaRq9rnUkpqlLg03Byfl/+M32mpxjO6KaPw==", + "dev": true, + "requires": { + "@babel/helper-module-transforms": "^7.17.7", + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/helper-simple-access": "^7.17.7", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-systemjs": { + "version": "7.17.8", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.17.8.tgz", + "integrity": "sha512-39reIkMTUVagzgA5x88zDYXPCMT6lcaRKs1+S9K6NKBPErbgO/w/kP8GlNQTC87b412ZTlmNgr3k2JrWgHH+Bw==", + "dev": true, + "requires": { + "@babel/helper-hoist-variables": "^7.16.7", + "@babel/helper-module-transforms": "^7.17.7", + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/helper-validator-identifier": "^7.16.7", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-umd": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.16.7.tgz", + "integrity": "sha512-EMh7uolsC8O4xhudF2F6wedbSHm1HHZ0C6aJ7K67zcDNidMzVcxWdGr+htW9n21klm+bOn+Rx4CBsAntZd3rEQ==", + "dev": true, + "requires": { + "@babel/helper-module-transforms": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.16.8", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.16.8.tgz", + "integrity": "sha512-j3Jw+n5PvpmhRR+mrgIh04puSANCk/T/UA3m3P1MjJkhlK906+ApHhDIqBQDdOgL/r1UYpz4GNclTXxyZrYGSw==", + "dev": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.16.7" + } + }, + "@babel/plugin-transform-new-target": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.16.7.tgz", + "integrity": "sha512-xiLDzWNMfKoGOpc6t3U+etCE2yRnn3SM09BXqWPIZOBpL2gvVrBWUKnsJx0K/ADi5F5YC5f8APFfWrz25TdlGg==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-object-super": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.16.7.tgz", + "integrity": "sha512-14J1feiQVWaGvRxj2WjyMuXS2jsBkgB3MdSN5HuC2G5nRspa5RK9COcs82Pwy5BuGcjb+fYaUj94mYcOj7rCvw==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/helper-replace-supers": "^7.16.7" + } + }, + "@babel/plugin-transform-parameters": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.16.7.tgz", + "integrity": "sha512-AT3MufQ7zZEhU2hwOA11axBnExW0Lszu4RL/tAlUJBuNoRak+wehQW8h6KcXOcgjY42fHtDxswuMhMjFEuv/aw==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-regenerator": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.17.9.tgz", + "integrity": "sha512-Lc2TfbxR1HOyn/c6b4Y/b6NHoTb67n/IoWLxTu4kC7h4KQnWlhCq2S8Tx0t2SVvv5Uu87Hs+6JEJ5kt2tYGylQ==", + "dev": true, + "requires": { + "regenerator-transform": "^0.15.0" + } + }, + "@babel/plugin-transform-runtime": { + "version": "7.17.0", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.17.0.tgz", + "integrity": "sha512-fr7zPWnKXNc1xoHfrIU9mN/4XKX4VLZ45Q+oMhfsYIaHvg7mHgmhfOy/ckRWqDK7XF3QDigRpkh5DKq6+clE8A==", + "dev": true, + "requires": { + "@babel/helper-module-imports": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7", + "babel-plugin-polyfill-corejs2": "^0.3.0", + "babel-plugin-polyfill-corejs3": "^0.5.0", + "babel-plugin-polyfill-regenerator": "^0.3.0", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "@babel/plugin-transform-shorthand-properties": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.16.7.tgz", + "integrity": "sha512-hah2+FEnoRoATdIb05IOXf+4GzXYTq75TVhIn1PewihbpyrNWUt2JbudKQOETWw6QpLe+AIUpJ5MVLYTQbeeUg==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-spread": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.16.7.tgz", + "integrity": "sha512-+pjJpgAngb53L0iaA5gU/1MLXJIfXcYepLgXB3esVRf4fqmj8f2cxM3/FKaHsZms08hFQJkFccEWuIpm429TXg==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7", + "@babel/helper-skip-transparent-expression-wrappers": "^7.16.0" + } + }, + "@babel/plugin-transform-sticky-regex": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.16.7.tgz", + "integrity": "sha512-NJa0Bd/87QV5NZZzTuZG5BPJjLYadeSZ9fO6oOUoL4iQx+9EEuw/eEM92SrsT19Yc2jgB1u1hsjqDtH02c3Drw==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-template-literals": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.16.7.tgz", + "integrity": "sha512-VwbkDDUeenlIjmfNeDX/V0aWrQH2QiVyJtwymVQSzItFDTpxfyJh3EVaQiS0rIN/CqbLGr0VcGmuwyTdZtdIsA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-typeof-symbol": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.16.7.tgz", + "integrity": "sha512-p2rOixCKRJzpg9JB4gjnG4gjWkWa89ZoYUnl9snJ1cWIcTH/hvxZqfO+WjG6T8DRBpctEol5jw1O5rA8gkCokQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/plugin-transform-unicode-regex": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.16.7.tgz", + "integrity": "sha512-oC5tYYKw56HO75KZVLQ+R/Nl3Hro9kf8iG0hXoaHP7tjAyCpvqBiSNe6vGrZni1Z6MggmUOC6A7VP7AVmw225Q==", + "dev": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.16.7", + "@babel/helper-plugin-utils": "^7.16.7" + } + }, + "@babel/preset-env": { + "version": "7.3.4", + "resolved": "https://registry.npmmirror.com/@babel/preset-env/-/preset-env-7.3.4.tgz", + "integrity": "sha512-2mwqfYMK8weA0g0uBKOt4FE3iEodiHy9/CW0b+nWXcbL+pGzLx8ESYc+j9IIxr6LTDHWKgPm71i9smo02bw+gA==", + "dev": true, + "requires": { + "@babel/helper-module-imports": "^7.0.0", + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/plugin-proposal-async-generator-functions": "^7.2.0", + "@babel/plugin-proposal-json-strings": "^7.2.0", + "@babel/plugin-proposal-object-rest-spread": "^7.3.4", + "@babel/plugin-proposal-optional-catch-binding": "^7.2.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.2.0", + "@babel/plugin-syntax-async-generators": "^7.2.0", + "@babel/plugin-syntax-json-strings": "^7.2.0", + "@babel/plugin-syntax-object-rest-spread": "^7.2.0", + "@babel/plugin-syntax-optional-catch-binding": "^7.2.0", + "@babel/plugin-transform-arrow-functions": "^7.2.0", + "@babel/plugin-transform-async-to-generator": "^7.3.4", + "@babel/plugin-transform-block-scoped-functions": "^7.2.0", + "@babel/plugin-transform-block-scoping": "^7.3.4", + "@babel/plugin-transform-classes": "^7.3.4", + "@babel/plugin-transform-computed-properties": "^7.2.0", + "@babel/plugin-transform-destructuring": "^7.2.0", + "@babel/plugin-transform-dotall-regex": "^7.2.0", + "@babel/plugin-transform-duplicate-keys": "^7.2.0", + "@babel/plugin-transform-exponentiation-operator": "^7.2.0", + "@babel/plugin-transform-for-of": "^7.2.0", + "@babel/plugin-transform-function-name": "^7.2.0", + "@babel/plugin-transform-literals": "^7.2.0", + "@babel/plugin-transform-modules-amd": "^7.2.0", + "@babel/plugin-transform-modules-commonjs": "^7.2.0", + "@babel/plugin-transform-modules-systemjs": "^7.3.4", + "@babel/plugin-transform-modules-umd": "^7.2.0", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.3.0", + "@babel/plugin-transform-new-target": "^7.0.0", + "@babel/plugin-transform-object-super": "^7.2.0", + "@babel/plugin-transform-parameters": "^7.2.0", + "@babel/plugin-transform-regenerator": "^7.3.4", + "@babel/plugin-transform-shorthand-properties": "^7.2.0", + "@babel/plugin-transform-spread": "^7.2.0", + "@babel/plugin-transform-sticky-regex": "^7.2.0", + "@babel/plugin-transform-template-literals": "^7.2.0", + "@babel/plugin-transform-typeof-symbol": "^7.2.0", + "@babel/plugin-transform-unicode-regex": "^7.2.0", + "browserslist": "^4.3.4", + "invariant": "^2.2.2", + "js-levenshtein": "^1.1.3", + "semver": "^5.3.0" + } + }, + "@babel/runtime": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/runtime/-/runtime-7.17.9.tgz", + "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", + "dev": true, + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, + "@babel/runtime-corejs2": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/runtime-corejs2/-/runtime-corejs2-7.17.9.tgz", + "integrity": "sha512-+QThIsnjVY12uURTvmnW33risFZ7ulq6OWw0VJL08UwiYiWVp9PM63s+W1L2ppajYyKAYKb7afcGYSHzA0k04Q==", + "dev": true, + "requires": { + "core-js": "^2.6.5", + "regenerator-runtime": "^0.13.4" + } + }, + "@babel/template": { + "version": "7.16.7", + "resolved": "https://registry.npmmirror.com/@babel/template/-/template-7.16.7.tgz", + "integrity": "sha512-I8j/x8kHUrbYRTUxXrrMbfCa7jxkE7tZre39x3kjr9hvI82cK1FfqLygotcWN5kdPGWcLdWMHpSBavse5tWw3w==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.16.7", + "@babel/parser": "^7.16.7", + "@babel/types": "^7.16.7" + } + }, + "@babel/traverse": { + "version": "7.17.9", + "resolved": "https://registry.npmmirror.com/@babel/traverse/-/traverse-7.17.9.tgz", + "integrity": "sha512-PQO8sDIJ8SIwipTPiR71kJQCKQYB5NGImbOviK8K+kg5xkNSYXLBupuX9QhatFowrsvo9Hj8WgArg3W7ijNAQw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.16.7", + "@babel/generator": "^7.17.9", + "@babel/helper-environment-visitor": "^7.16.7", + "@babel/helper-function-name": "^7.17.9", + "@babel/helper-hoist-variables": "^7.16.7", + "@babel/helper-split-export-declaration": "^7.16.7", + "@babel/parser": "^7.17.9", + "@babel/types": "^7.17.0", + "debug": "^4.1.0", + "globals": "^11.1.0" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "@babel/types": { + "version": "7.17.0", + "resolved": "https://registry.npmmirror.com/@babel/types/-/types-7.17.0.tgz", + "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.16.7", + "to-fast-properties": "^2.0.0" + } + }, + "@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "dev": true + }, + "@hapi/address": { + "version": "2.1.4", + "resolved": "https://registry.npmmirror.com/@hapi/address/-/address-2.1.4.tgz", + "integrity": "sha512-QD1PhQk+s31P1ixsX0H0Suoupp3VMXzIVMSwobR3F3MSUO2YCV0B7xqLcUw/Bh8yuvd3LhpyqLQWTNcRmp6IdQ==", + "dev": true + }, + "@hapi/bourne": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/@hapi/bourne/-/bourne-1.3.2.tgz", + "integrity": "sha512-1dVNHT76Uu5N3eJNTYcvxee+jzX4Z9lfciqRRHCU27ihbUcYi+iSc2iml5Ke1LXe1SyJCLA0+14Jh4tXJgOppA==", + "dev": true + }, + "@hapi/hoek": { + "version": "8.5.1", + "resolved": "https://registry.npmmirror.com/@hapi/hoek/-/hoek-8.5.1.tgz", + "integrity": "sha512-yN7kbciD87WzLGc5539Tn0sApjyiGHAJgKvG9W8C7O+6c7qmoQMfVs0W4bX17eqz6C78QJqqFrtgdK5EWf6Qow==", + "dev": true + }, + "@hapi/joi": { + "version": "15.1.1", + "resolved": "https://registry.npmmirror.com/@hapi/joi/-/joi-15.1.1.tgz", + "integrity": "sha512-entf8ZMOK8sc+8YfeOlM8pCfg3b5+WZIKBfUaaJT8UsjAAPjartzxIYm3TIbjvA4u+u++KbcXD38k682nVHDAQ==", + "dev": true, + "requires": { + "@hapi/address": "2.x.x", + "@hapi/bourne": "1.x.x", + "@hapi/hoek": "8.x.x", + "@hapi/topo": "3.x.x" + } + }, + "@hapi/topo": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/@hapi/topo/-/topo-3.1.6.tgz", + "integrity": "sha512-tAag0jEcjwH+P2quUfipd7liWCNX2F8NvYjQp2wtInsZxnMlypdw0FtAOLxtvvkO+GSRRbmNi8m/5y42PQJYCQ==", + "dev": true, + "requires": { + "@hapi/hoek": "^8.3.0" + } + }, + "@hutson/parse-repository-url": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@hutson/parse-repository-url/-/parse-repository-url-3.0.2.tgz", + "integrity": "sha512-H9XAx3hc0BQHY6l+IFSWHDySypcXsvsuLhgYLUGywmJ5pswRVQJUHpOsobnLYp2ZUaUlKiKDrgWWhosOwAEM8Q==", + "dev": true + }, + "@intervolga/optimize-cssnano-plugin": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/@intervolga/optimize-cssnano-plugin/-/optimize-cssnano-plugin-1.0.6.tgz", + "integrity": "sha512-zN69TnSr0viRSU6cEDIcuPcP67QcpQ6uHACg58FiN9PDrU6SLyGW3MR4tiISbYxy1kDWAVPwD+XwQTWE5cigAA==", + "dev": true, + "requires": { + "cssnano": "^4.0.0", + "cssnano-preset-default": "^4.0.0", + "postcss": "^7.0.0" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/@jridgewell/resolve-uri/-/resolve-uri-3.0.6.tgz", + "integrity": "sha512-R7xHtBSNm+9SyvpJkdQl+qrM3Hm2fea3Ef197M3mUug+v+yR+Rhfbs7PBtcBUVnIWJ4JcAdjvij+c8hXS9p5aw==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.11", + "resolved": "https://registry.npmmirror.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.11.tgz", + "integrity": "sha512-Fg32GrJo61m+VqYSdRSjRXMjQ06j8YIYfcTqndLYVAaHmroZHLJZCydsWBOTDqXS2v+mjxohBWEMfg97GXmYQg==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmmirror.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "@kazupon/vue-i18n-loader": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/@kazupon/vue-i18n-loader/-/vue-i18n-loader-0.4.1.tgz", + "integrity": "sha512-hVznmhnyoUKozGY7pwq/UtPL76UDzb+aiN2YksZZIzCY/MkEqih0MSyEmTGw7+HVWzJRPAlDyoRNR4tWKmkCRw==", + "dev": true, + "requires": { + "js-yaml": "^3.13.1", + "json5": "^2.1.0" + }, + "dependencies": { + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true + } + } + }, + "@lerna/add": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/add/-/add-4.0.0.tgz", + "integrity": "sha512-cpmAH1iS3k8JBxNvnMqrGTTjbY/ZAiKa1ChJzFevMYY3eeqbvhsBKnBcxjRXtdrJ6bd3dCQM+ZtK+0i682Fhng==", + "dev": true, + "requires": { + "@lerna/bootstrap": "4.0.0", + "@lerna/command": "4.0.0", + "@lerna/filter-options": "4.0.0", + "@lerna/npm-conf": "4.0.0", + "@lerna/validation-error": "4.0.0", + "dedent": "^0.7.0", + "npm-package-arg": "^8.1.0", + "p-map": "^4.0.0", + "pacote": "^11.2.6", + "semver": "^7.3.4" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/bootstrap": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/bootstrap/-/bootstrap-4.0.0.tgz", + "integrity": "sha512-RkS7UbeM2vu+kJnHzxNRCLvoOP9yGNgkzRdy4UV2hNalD7EP41bLvRVOwRYQ7fhc2QcbhnKNdOBihYRL0LcKtw==", + "dev": true, + "requires": { + "@lerna/command": "4.0.0", + "@lerna/filter-options": "4.0.0", + "@lerna/has-npm-version": "4.0.0", + "@lerna/npm-install": "4.0.0", + "@lerna/package-graph": "4.0.0", + "@lerna/pulse-till-done": "4.0.0", + "@lerna/rimraf-dir": "4.0.0", + "@lerna/run-lifecycle": "4.0.0", + "@lerna/run-topologically": "4.0.0", + "@lerna/symlink-binary": "4.0.0", + "@lerna/symlink-dependencies": "4.0.0", + "@lerna/validation-error": "4.0.0", + "dedent": "^0.7.0", + "get-port": "^5.1.1", + "multimatch": "^5.0.0", + "npm-package-arg": "^8.1.0", + "npmlog": "^4.1.2", + "p-map": "^4.0.0", + "p-map-series": "^2.1.0", + "p-waterfall": "^2.1.1", + "read-package-tree": "^5.3.1", + "semver": "^7.3.4" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/changed": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/changed/-/changed-4.0.0.tgz", + "integrity": "sha512-cD+KuPRp6qiPOD+BO6S6SN5cARspIaWSOqGBpGnYzLb4uWT8Vk4JzKyYtc8ym1DIwyoFXHosXt8+GDAgR8QrgQ==", + "dev": true, + "requires": { + "@lerna/collect-updates": "4.0.0", + "@lerna/command": "4.0.0", + "@lerna/listable": "4.0.0", + "@lerna/output": "4.0.0" + } + }, + "@lerna/check-working-tree": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/check-working-tree/-/check-working-tree-4.0.0.tgz", + "integrity": "sha512-/++bxM43jYJCshBiKP5cRlCTwSJdRSxVmcDAXM+1oUewlZJVSVlnks5eO0uLxokVFvLhHlC5kHMc7gbVFPHv6Q==", + "dev": true, + "requires": { + "@lerna/collect-uncommitted": "4.0.0", + "@lerna/describe-ref": "4.0.0", + "@lerna/validation-error": "4.0.0" + } + }, + "@lerna/child-process": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/child-process/-/child-process-4.0.0.tgz", + "integrity": "sha512-XtCnmCT9eyVsUUHx6y/CTBYdV9g2Cr/VxyseTWBgfIur92/YKClfEtJTbOh94jRT62hlKLqSvux/UhxXVh613Q==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "execa": "^5.0.0", + "strong-log-transformer": "^2.1.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "execa": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + } + }, + "get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true + }, + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "requires": { + "path-key": "^3.0.0" + } + }, + "onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, + "@lerna/clean": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/clean/-/clean-4.0.0.tgz", + "integrity": "sha512-uugG2iN9k45ITx2jtd8nEOoAtca8hNlDCUM0N3lFgU/b1mEQYAPRkqr1qs4FLRl/Y50ZJ41wUz1eazS+d/0osA==", + "dev": true, + "requires": { + "@lerna/command": "4.0.0", + "@lerna/filter-options": "4.0.0", + "@lerna/prompt": "4.0.0", + "@lerna/pulse-till-done": "4.0.0", + "@lerna/rimraf-dir": "4.0.0", + "p-map": "^4.0.0", + "p-map-series": "^2.1.0", + "p-waterfall": "^2.1.1" + }, + "dependencies": { + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + } + } + }, + "@lerna/cli": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/cli/-/cli-4.0.0.tgz", + "integrity": "sha512-Neaw3GzFrwZiRZv2g7g6NwFjs3er1vhraIniEs0jjVLPMNC4eata0na3GfE5yibkM/9d3gZdmihhZdZ3EBdvYA==", + "dev": true, + "requires": { + "@lerna/global-options": "4.0.0", + "dedent": "^0.7.0", + "npmlog": "^4.1.2", + "yargs": "^16.2.0" + } + }, + "@lerna/collect-uncommitted": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/collect-uncommitted/-/collect-uncommitted-4.0.0.tgz", + "integrity": "sha512-ufSTfHZzbx69YNj7KXQ3o66V4RC76ffOjwLX0q/ab//61bObJ41n03SiQEhSlmpP+gmFbTJ3/7pTe04AHX9m/g==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "chalk": "^4.1.0", + "npmlog": "^4.1.2" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "@lerna/collect-updates": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/collect-updates/-/collect-updates-4.0.0.tgz", + "integrity": "sha512-bnNGpaj4zuxsEkyaCZLka9s7nMs58uZoxrRIPJ+nrmrZYp1V5rrd+7/NYTuunOhY2ug1sTBvTAxj3NZQ+JKnOw==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/describe-ref": "4.0.0", + "minimatch": "^3.0.4", + "npmlog": "^4.1.2", + "slash": "^3.0.0" + }, + "dependencies": { + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + } + } + }, + "@lerna/command": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/command/-/command-4.0.0.tgz", + "integrity": "sha512-LM9g3rt5FsPNFqIHUeRwWXLNHJ5NKzOwmVKZ8anSp4e1SPrv2HNc1V02/9QyDDZK/w+5POXH5lxZUI1CHaOK/A==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/package-graph": "4.0.0", + "@lerna/project": "4.0.0", + "@lerna/validation-error": "4.0.0", + "@lerna/write-log-file": "4.0.0", + "clone-deep": "^4.0.1", + "dedent": "^0.7.0", + "execa": "^5.0.0", + "is-ci": "^2.0.0", + "npmlog": "^4.1.2" + }, + "dependencies": { + "ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "dev": true + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "execa": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + } + }, + "get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true + }, + "human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true + }, + "is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dev": true, + "requires": { + "ci-info": "^2.0.0" + } + }, + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "requires": { + "path-key": "^3.0.0" + } + }, + "onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, + "@lerna/conventional-commits": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/conventional-commits/-/conventional-commits-4.0.0.tgz", + "integrity": "sha512-CSUQRjJHFrH8eBn7+wegZLV3OrNc0Y1FehYfYGhjLE2SIfpCL4bmfu/ViYuHh9YjwHaA+4SX6d3hR+xkeseKmw==", + "dev": true, + "requires": { + "@lerna/validation-error": "4.0.0", + "conventional-changelog-angular": "^5.0.12", + "conventional-changelog-core": "^4.2.2", + "conventional-recommended-bump": "^6.1.0", + "fs-extra": "^9.1.0", + "get-stream": "^6.0.0", + "lodash.template": "^4.5.0", + "npm-package-arg": "^8.1.0", + "npmlog": "^4.1.2", + "pify": "^5.0.0", + "semver": "^7.3.4" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==", + "dev": true + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/create": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/create/-/create-4.0.0.tgz", + "integrity": "sha512-mVOB1niKByEUfxlbKTM1UNECWAjwUdiioIbRQZEeEabtjCL69r9rscIsjlGyhGWCfsdAG5wfq4t47nlDXdLLag==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/command": "4.0.0", + "@lerna/npm-conf": "4.0.0", + "@lerna/validation-error": "4.0.0", + "dedent": "^0.7.0", + "fs-extra": "^9.1.0", + "globby": "^11.0.2", + "init-package-json": "^2.0.2", + "npm-package-arg": "^8.1.0", + "p-reduce": "^2.1.0", + "pacote": "^11.2.6", + "pify": "^5.0.0", + "semver": "^7.3.4", + "slash": "^3.0.0", + "validate-npm-package-license": "^3.0.4", + "validate-npm-package-name": "^3.0.0", + "whatwg-url": "^8.4.0", + "yargs-parser": "20.2.4" + }, + "dependencies": { + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true + }, + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "requires": { + "path-type": "^4.0.0" + } + }, + "fast-glob": { + "version": "3.2.11", + "resolved": "https://registry.npmmirror.com/fast-glob/-/fast-glob-3.2.11.tgz", + "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "globby": { + "version": "11.1.0", + "resolved": "https://registry.npmmirror.com/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + } + }, + "ignore": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "dev": true + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmmirror.com/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dev": true, + "requires": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + } + }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==", + "dev": true + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "yargs-parser": { + "version": "20.2.4", + "resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "dev": true + } + } + }, + "@lerna/create-symlink": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/create-symlink/-/create-symlink-4.0.0.tgz", + "integrity": "sha512-I0phtKJJdafUiDwm7BBlEUOtogmu8+taxq6PtIrxZbllV9hWg59qkpuIsiFp+no7nfRVuaasNYHwNUhDAVQBig==", + "dev": true, + "requires": { + "cmd-shim": "^4.1.0", + "fs-extra": "^9.1.0", + "npmlog": "^4.1.2" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + } + } + }, + "@lerna/describe-ref": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/describe-ref/-/describe-ref-4.0.0.tgz", + "integrity": "sha512-eTU5+xC4C5Gcgz+Ey4Qiw9nV2B4JJbMulsYJMW8QjGcGh8zudib7Sduj6urgZXUYNyhYpRs+teci9M2J8u+UvQ==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "npmlog": "^4.1.2" + } + }, + "@lerna/diff": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/diff/-/diff-4.0.0.tgz", + "integrity": "sha512-jYPKprQVg41+MUMxx6cwtqsNm0Yxx9GDEwdiPLwcUTFx+/qKCEwifKNJ1oGIPBxyEHX2PFCOjkK39lHoj2qiag==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/command": "4.0.0", + "@lerna/validation-error": "4.0.0", + "npmlog": "^4.1.2" + } + }, + "@lerna/exec": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/exec/-/exec-4.0.0.tgz", + "integrity": "sha512-VGXtL/b/JfY84NB98VWZpIExfhLOzy0ozm/0XaS4a2SmkAJc5CeUfrhvHxxkxiTBLkU+iVQUyYEoAT0ulQ8PCw==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/command": "4.0.0", + "@lerna/filter-options": "4.0.0", + "@lerna/profiler": "4.0.0", + "@lerna/run-topologically": "4.0.0", + "@lerna/validation-error": "4.0.0", + "p-map": "^4.0.0" + }, + "dependencies": { + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + } + } + }, + "@lerna/filter-options": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/filter-options/-/filter-options-4.0.0.tgz", + "integrity": "sha512-vV2ANOeZhOqM0rzXnYcFFCJ/kBWy/3OA58irXih9AMTAlQLymWAK0akWybl++sUJ4HB9Hx12TOqaXbYS2NM5uw==", + "dev": true, + "requires": { + "@lerna/collect-updates": "4.0.0", + "@lerna/filter-packages": "4.0.0", + "dedent": "^0.7.0", + "npmlog": "^4.1.2" + } + }, + "@lerna/filter-packages": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/filter-packages/-/filter-packages-4.0.0.tgz", + "integrity": "sha512-+4AJIkK7iIiOaqCiVTYJxh/I9qikk4XjNQLhE3kixaqgMuHl1NQ99qXRR0OZqAWB9mh8Z1HA9bM5K1HZLBTOqA==", + "dev": true, + "requires": { + "@lerna/validation-error": "4.0.0", + "multimatch": "^5.0.0", + "npmlog": "^4.1.2" + } + }, + "@lerna/get-npm-exec-opts": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/get-npm-exec-opts/-/get-npm-exec-opts-4.0.0.tgz", + "integrity": "sha512-yvmkerU31CTWS2c7DvmAWmZVeclPBqI7gPVr5VATUKNWJ/zmVcU4PqbYoLu92I9Qc4gY1TuUplMNdNuZTSL7IQ==", + "dev": true, + "requires": { + "npmlog": "^4.1.2" + } + }, + "@lerna/get-packed": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/get-packed/-/get-packed-4.0.0.tgz", + "integrity": "sha512-rfWONRsEIGyPJTxFzC8ECb3ZbsDXJbfqWYyeeQQDrJRPnEJErlltRLPLgC2QWbxFgFPsoDLeQmFHJnf0iDfd8w==", + "dev": true, + "requires": { + "fs-extra": "^9.1.0", + "ssri": "^8.0.1", + "tar": "^6.1.0" + }, + "dependencies": { + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "dev": true, + "requires": { + "minipass": "^3.1.1" + } + }, + "tar": { + "version": "6.1.11", + "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/github-client": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/github-client/-/github-client-4.0.0.tgz", + "integrity": "sha512-2jhsldZtTKXYUBnOm23Lb0Fx8G4qfSXF9y7UpyUgWUj+YZYd+cFxSuorwQIgk5P4XXrtVhsUesIsli+BYSThiw==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@octokit/plugin-enterprise-rest": "^6.0.1", + "@octokit/rest": "^18.1.0", + "git-url-parse": "^11.4.4", + "npmlog": "^4.1.2" + } + }, + "@lerna/gitlab-client": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/gitlab-client/-/gitlab-client-4.0.0.tgz", + "integrity": "sha512-OMUpGSkeDWFf7BxGHlkbb35T7YHqVFCwBPSIR6wRsszY8PAzCYahtH3IaJzEJyUg6vmZsNl0FSr3pdA2skhxqA==", + "dev": true, + "requires": { + "node-fetch": "^2.6.1", + "npmlog": "^4.1.2", + "whatwg-url": "^8.4.0" + } + }, + "@lerna/global-options": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/global-options/-/global-options-4.0.0.tgz", + "integrity": "sha512-TRMR8afAHxuYBHK7F++Ogop2a82xQjoGna1dvPOY6ltj/pEx59pdgcJfYcynYqMkFIk8bhLJJN9/ndIfX29FTQ==", + "dev": true + }, + "@lerna/has-npm-version": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/has-npm-version/-/has-npm-version-4.0.0.tgz", + "integrity": "sha512-LQ3U6XFH8ZmLCsvsgq1zNDqka0Xzjq5ibVN+igAI5ccRWNaUsE/OcmsyMr50xAtNQMYMzmpw5GVLAivT2/YzCg==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "semver": "^7.3.4" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/import": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/import/-/import-4.0.0.tgz", + "integrity": "sha512-FaIhd+4aiBousKNqC7TX1Uhe97eNKf5/SC7c5WZANVWtC7aBWdmswwDt3usrzCNpj6/Wwr9EtEbYROzxKH8ffg==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/command": "4.0.0", + "@lerna/prompt": "4.0.0", + "@lerna/pulse-till-done": "4.0.0", + "@lerna/validation-error": "4.0.0", + "dedent": "^0.7.0", + "fs-extra": "^9.1.0", + "p-map-series": "^2.1.0" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + } + } + }, + "@lerna/info": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/info/-/info-4.0.0.tgz", + "integrity": "sha512-8Uboa12kaCSZEn4XRfPz5KU9XXoexSPS4oeYGj76s2UQb1O1GdnEyfjyNWoUl1KlJ2i/8nxUskpXIftoFYH0/Q==", + "dev": true, + "requires": { + "@lerna/command": "4.0.0", + "@lerna/output": "4.0.0", + "envinfo": "^7.7.4" + } + }, + "@lerna/init": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/init/-/init-4.0.0.tgz", + "integrity": "sha512-wY6kygop0BCXupzWj5eLvTUqdR7vIAm0OgyV9WHpMYQGfs1V22jhztt8mtjCloD/O0nEe4tJhdG62XU5aYmPNQ==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/command": "4.0.0", + "fs-extra": "^9.1.0", + "p-map": "^4.0.0", + "write-json-file": "^4.3.0" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + } + } + }, + "@lerna/link": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/link/-/link-4.0.0.tgz", + "integrity": "sha512-KlvPi7XTAcVOByfaLlOeYOfkkDcd+bejpHMCd1KcArcFTwijOwXOVi24DYomIeHvy6HsX/IUquJ4PPUJIeB4+w==", + "dev": true, + "requires": { + "@lerna/command": "4.0.0", + "@lerna/package-graph": "4.0.0", + "@lerna/symlink-dependencies": "4.0.0", + "p-map": "^4.0.0", + "slash": "^3.0.0" + }, + "dependencies": { + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + } + } + }, + "@lerna/list": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/list/-/list-4.0.0.tgz", + "integrity": "sha512-L2B5m3P+U4Bif5PultR4TI+KtW+SArwq1i75QZ78mRYxPc0U/piau1DbLOmwrdqr99wzM49t0Dlvl6twd7GHFg==", + "dev": true, + "requires": { + "@lerna/command": "4.0.0", + "@lerna/filter-options": "4.0.0", + "@lerna/listable": "4.0.0", + "@lerna/output": "4.0.0" + } + }, + "@lerna/listable": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/listable/-/listable-4.0.0.tgz", + "integrity": "sha512-/rPOSDKsOHs5/PBLINZOkRIX1joOXUXEtyUs5DHLM8q6/RP668x/1lFhw6Dx7/U+L0+tbkpGtZ1Yt0LewCLgeQ==", + "dev": true, + "requires": { + "@lerna/query-graph": "4.0.0", + "chalk": "^4.1.0", + "columnify": "^1.5.4" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "@lerna/log-packed": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/log-packed/-/log-packed-4.0.0.tgz", + "integrity": "sha512-+dpCiWbdzgMAtpajLToy9PO713IHoE6GV/aizXycAyA07QlqnkpaBNZ8DW84gHdM1j79TWockGJo9PybVhrrZQ==", + "dev": true, + "requires": { + "byte-size": "^7.0.0", + "columnify": "^1.5.4", + "has-unicode": "^2.0.1", + "npmlog": "^4.1.2" + } + }, + "@lerna/npm-conf": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/npm-conf/-/npm-conf-4.0.0.tgz", + "integrity": "sha512-uS7H02yQNq3oejgjxAxqq/jhwGEE0W0ntr8vM3EfpCW1F/wZruwQw+7bleJQ9vUBjmdXST//tk8mXzr5+JXCfw==", + "dev": true, + "requires": { + "config-chain": "^1.1.12", + "pify": "^5.0.0" + }, + "dependencies": { + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==", + "dev": true + } + } + }, + "@lerna/npm-dist-tag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/npm-dist-tag/-/npm-dist-tag-4.0.0.tgz", + "integrity": "sha512-F20sg28FMYTgXqEQihgoqSfwmq+Id3zT23CnOwD+XQMPSy9IzyLf1fFVH319vXIw6NF6Pgs4JZN2Qty6/CQXGw==", + "dev": true, + "requires": { + "@lerna/otplease": "4.0.0", + "npm-package-arg": "^8.1.0", + "npm-registry-fetch": "^9.0.0", + "npmlog": "^4.1.2" + }, + "dependencies": { + "cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmmirror.com/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "dev": true, + "requires": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + } + }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "make-fetch-happen": { + "version": "8.0.14", + "resolved": "https://registry.npmmirror.com/make-fetch-happen/-/make-fetch-happen-8.0.14.tgz", + "integrity": "sha512-EsS89h6l4vbfJEtBZnENTOFk8mCRpY5ru36Xe5bcX1KYIli2mkSHqoFsp5O1wMDvTJJzxe/4THpCTtygjeeGWQ==", + "dev": true, + "requires": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.0.5", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^5.0.0", + "ssri": "^8.0.0" + } + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "npm-registry-fetch": { + "version": "9.0.0", + "resolved": "https://registry.npmmirror.com/npm-registry-fetch/-/npm-registry-fetch-9.0.0.tgz", + "integrity": "sha512-PuFYYtnQ8IyVl6ib9d3PepeehcUeHN9IO5N/iCRhyg9tStQcqGQBRVHmfmMWPDERU3KwZoHFvbJ4FPXPspvzbA==", + "dev": true, + "requires": { + "@npmcli/ci-detect": "^1.0.0", + "lru-cache": "^6.0.0", + "make-fetch-happen": "^8.0.9", + "minipass": "^3.1.3", + "minipass-fetch": "^1.3.0", + "minipass-json-stream": "^1.0.1", + "minizlib": "^2.0.0", + "npm-package-arg": "^8.0.0" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "socks-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/socks-proxy-agent/-/socks-proxy-agent-5.0.1.tgz", + "integrity": "sha512-vZdmnjb9a2Tz6WEQVIurybSwElwPxMZaIc7PzqbJTrezcKNznv6giT7J7tZDZ1BojVaa1jvO/UiUdhDVB0ACoQ==", + "dev": true, + "requires": { + "agent-base": "^6.0.2", + "debug": "4", + "socks": "^2.3.3" + } + }, + "ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "dev": true, + "requires": { + "minipass": "^3.1.1" + } + }, + "tar": { + "version": "6.1.11", + "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/npm-install": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/npm-install/-/npm-install-4.0.0.tgz", + "integrity": "sha512-aKNxq2j3bCH3eXl3Fmu4D54s/YLL9WSwV8W7X2O25r98wzrO38AUN6AB9EtmAx+LV/SP15et7Yueg9vSaanRWg==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/get-npm-exec-opts": "4.0.0", + "fs-extra": "^9.1.0", + "npm-package-arg": "^8.1.0", + "npmlog": "^4.1.2", + "signal-exit": "^3.0.3", + "write-pkg": "^4.0.0" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + } + } + }, + "@lerna/npm-publish": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/npm-publish/-/npm-publish-4.0.0.tgz", + "integrity": "sha512-vQb7yAPRo5G5r77DRjHITc9piR9gvEKWrmfCH7wkfBnGWEqu7n8/4bFQ7lhnkujvc8RXOsYpvbMQkNfkYibD/w==", + "dev": true, + "requires": { + "@lerna/otplease": "4.0.0", + "@lerna/run-lifecycle": "4.0.0", + "fs-extra": "^9.1.0", + "libnpmpublish": "^4.0.0", + "npm-package-arg": "^8.1.0", + "npmlog": "^4.1.2", + "pify": "^5.0.0", + "read-package-json": "^3.0.0" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "normalize-package-data": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", + "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", + "dev": true, + "requires": { + "hosted-git-info": "^4.0.1", + "is-core-module": "^2.5.0", + "semver": "^7.3.4", + "validate-npm-package-license": "^3.0.1" + } + }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==", + "dev": true + }, + "read-package-json": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/read-package-json/-/read-package-json-3.0.1.tgz", + "integrity": "sha512-aLcPqxovhJTVJcsnROuuzQvv6oziQx4zd3JvG0vGCL5MjTONUc4uJ90zCBC6R7W7oUKBNoR/F8pkyfVwlbxqng==", + "dev": true, + "requires": { + "glob": "^7.1.1", + "json-parse-even-better-errors": "^2.3.0", + "normalize-package-data": "^3.0.0", + "npm-normalize-package-bin": "^1.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/npm-run-script": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/npm-run-script/-/npm-run-script-4.0.0.tgz", + "integrity": "sha512-Jmyh9/IwXJjOXqKfIgtxi0bxi1pUeKe5bD3S81tkcy+kyng/GNj9WSqD5ZggoNP2NP//s4CLDAtUYLdP7CU9rA==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "@lerna/get-npm-exec-opts": "4.0.0", + "npmlog": "^4.1.2" + } + }, + "@lerna/otplease": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/otplease/-/otplease-4.0.0.tgz", + "integrity": "sha512-Sgzbqdk1GH4psNiT6hk+BhjOfIr/5KhGBk86CEfHNJTk9BK4aZYyJD4lpDbDdMjIV4g03G7pYoqHzH765T4fxw==", + "dev": true, + "requires": { + "@lerna/prompt": "4.0.0" + } + }, + "@lerna/output": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/output/-/output-4.0.0.tgz", + "integrity": "sha512-Un1sHtO1AD7buDQrpnaYTi2EG6sLF+KOPEAMxeUYG5qG3khTs2Zgzq5WE3dt2N/bKh7naESt20JjIW6tBELP0w==", + "dev": true, + "requires": { + "npmlog": "^4.1.2" + } + }, + "@lerna/pack-directory": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/pack-directory/-/pack-directory-4.0.0.tgz", + "integrity": "sha512-NJrmZNmBHS+5aM+T8N6FVbaKFScVqKlQFJNY2k7nsJ/uklNKsLLl6VhTQBPwMTbf6Tf7l6bcKzpy7aePuq9UiQ==", + "dev": true, + "requires": { + "@lerna/get-packed": "4.0.0", + "@lerna/package": "4.0.0", + "@lerna/run-lifecycle": "4.0.0", + "npm-packlist": "^2.1.4", + "npmlog": "^4.1.2", + "tar": "^6.1.0", + "temp-write": "^4.0.0" + }, + "dependencies": { + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "tar": { + "version": "6.1.11", + "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/package": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/package/-/package-4.0.0.tgz", + "integrity": "sha512-l0M/izok6FlyyitxiQKr+gZLVFnvxRQdNhzmQ6nRnN9dvBJWn+IxxpM+cLqGACatTnyo9LDzNTOj2Db3+s0s8Q==", + "dev": true, + "requires": { + "load-json-file": "^6.2.0", + "npm-package-arg": "^8.1.0", + "write-pkg": "^4.0.0" + } + }, + "@lerna/package-graph": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/package-graph/-/package-graph-4.0.0.tgz", + "integrity": "sha512-QED2ZCTkfXMKFoTGoccwUzjHtZMSf3UKX14A4/kYyBms9xfFsesCZ6SLI5YeySEgcul8iuIWfQFZqRw+Qrjraw==", + "dev": true, + "requires": { + "@lerna/prerelease-id-from-version": "4.0.0", + "@lerna/validation-error": "4.0.0", + "npm-package-arg": "^8.1.0", + "npmlog": "^4.1.2", + "semver": "^7.3.4" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/prerelease-id-from-version": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/prerelease-id-from-version/-/prerelease-id-from-version-4.0.0.tgz", + "integrity": "sha512-GQqguzETdsYRxOSmdFZ6zDBXDErIETWOqomLERRY54f4p+tk4aJjoVdd9xKwehC9TBfIFvlRbL1V9uQGHh1opg==", + "dev": true, + "requires": { + "semver": "^7.3.4" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/profiler": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/profiler/-/profiler-4.0.0.tgz", + "integrity": "sha512-/BaEbqnVh1LgW/+qz8wCuI+obzi5/vRE8nlhjPzdEzdmWmZXuCKyWSEzAyHOJWw1ntwMiww5dZHhFQABuoFz9Q==", + "dev": true, + "requires": { + "fs-extra": "^9.1.0", + "npmlog": "^4.1.2", + "upath": "^2.0.1" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + }, + "upath": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/upath/-/upath-2.0.1.tgz", + "integrity": "sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==", + "dev": true + } + } + }, + "@lerna/project": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/project/-/project-4.0.0.tgz", + "integrity": "sha512-o0MlVbDkD5qRPkFKlBZsXZjoNTWPyuL58564nSfZJ6JYNmgAptnWPB2dQlAc7HWRZkmnC2fCkEdoU+jioPavbg==", + "dev": true, + "requires": { + "@lerna/package": "4.0.0", + "@lerna/validation-error": "4.0.0", + "cosmiconfig": "^7.0.0", + "dedent": "^0.7.0", + "dot-prop": "^6.0.1", + "glob-parent": "^5.1.1", + "globby": "^11.0.2", + "load-json-file": "^6.2.0", + "npmlog": "^4.1.2", + "p-map": "^4.0.0", + "resolve-from": "^5.0.0", + "write-json-file": "^4.3.0" + }, + "dependencies": { + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true + }, + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "cosmiconfig": { + "version": "7.0.1", + "resolved": "https://registry.npmmirror.com/cosmiconfig/-/cosmiconfig-7.0.1.tgz", + "integrity": "sha512-a1YWNUV2HwGimB7dU2s1wUMurNKjpx60HxBB6xUM8Re+2s1g1IIfJvFR0/iCF+XHdE0GMTKTuLR32UQff4TEyQ==", + "dev": true, + "requires": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.2.1", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.10.0" + } + }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "requires": { + "path-type": "^4.0.0" + } + }, + "dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "dev": true, + "requires": { + "is-obj": "^2.0.0" + } + }, + "fast-glob": { + "version": "3.2.11", + "resolved": "https://registry.npmmirror.com/fast-glob/-/fast-glob-3.2.11.tgz", + "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "globby": { + "version": "11.1.0", + "resolved": "https://registry.npmmirror.com/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + } + }, + "ignore": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "dev": true + }, + "import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "dependencies": { + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + } + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmmirror.com/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dev": true, + "requires": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + } + }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + }, + "resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true + }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + } + } + }, + "@lerna/prompt": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/prompt/-/prompt-4.0.0.tgz", + "integrity": "sha512-4Ig46oCH1TH5M7YyTt53fT6TuaKMgqUUaqdgxvp6HP6jtdak6+amcsqB8YGz2eQnw/sdxunx84DfI9XpoLj4bQ==", + "dev": true, + "requires": { + "inquirer": "^7.3.3", + "npmlog": "^4.1.2" + }, + "dependencies": { + "ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmmirror.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "requires": { + "type-fest": "^0.21.3" + } + }, + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmmirror.com/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", + "dev": true + }, + "cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "requires": { + "restore-cursor": "^3.1.0" + } + }, + "cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "dev": true + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dev": true, + "requires": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + } + }, + "figures": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "inquirer": { + "version": "7.3.3", + "resolved": "https://registry.npmmirror.com/inquirer/-/inquirer-7.3.3.tgz", + "integrity": "sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==", + "dev": true, + "requires": { + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.19", + "mute-stream": "0.0.8", + "run-async": "^2.4.0", + "rxjs": "^6.6.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6" + } + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmmirror.com/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "dev": true + }, + "onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "requires": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + } + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, + "type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true + } + } + }, + "@lerna/publish": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/publish/-/publish-4.0.0.tgz", + "integrity": "sha512-K8jpqjHrChH22qtkytA5GRKIVFEtqBF6JWj1I8dWZtHs4Jywn8yB1jQ3BAMLhqmDJjWJtRck0KXhQQKzDK2UPg==", + "dev": true, + "requires": { + "@lerna/check-working-tree": "4.0.0", + "@lerna/child-process": "4.0.0", + "@lerna/collect-updates": "4.0.0", + "@lerna/command": "4.0.0", + "@lerna/describe-ref": "4.0.0", + "@lerna/log-packed": "4.0.0", + "@lerna/npm-conf": "4.0.0", + "@lerna/npm-dist-tag": "4.0.0", + "@lerna/npm-publish": "4.0.0", + "@lerna/otplease": "4.0.0", + "@lerna/output": "4.0.0", + "@lerna/pack-directory": "4.0.0", + "@lerna/prerelease-id-from-version": "4.0.0", + "@lerna/prompt": "4.0.0", + "@lerna/pulse-till-done": "4.0.0", + "@lerna/run-lifecycle": "4.0.0", + "@lerna/run-topologically": "4.0.0", + "@lerna/validation-error": "4.0.0", + "@lerna/version": "4.0.0", + "fs-extra": "^9.1.0", + "libnpmaccess": "^4.0.1", + "npm-package-arg": "^8.1.0", + "npm-registry-fetch": "^9.0.0", + "npmlog": "^4.1.2", + "p-map": "^4.0.0", + "p-pipe": "^3.1.0", + "pacote": "^11.2.6", + "semver": "^7.3.4" + }, + "dependencies": { + "cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmmirror.com/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "dev": true, + "requires": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + } + }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "make-fetch-happen": { + "version": "8.0.14", + "resolved": "https://registry.npmmirror.com/make-fetch-happen/-/make-fetch-happen-8.0.14.tgz", + "integrity": "sha512-EsS89h6l4vbfJEtBZnENTOFk8mCRpY5ru36Xe5bcX1KYIli2mkSHqoFsp5O1wMDvTJJzxe/4THpCTtygjeeGWQ==", + "dev": true, + "requires": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.0.5", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^5.0.0", + "ssri": "^8.0.0" + } + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "npm-registry-fetch": { + "version": "9.0.0", + "resolved": "https://registry.npmmirror.com/npm-registry-fetch/-/npm-registry-fetch-9.0.0.tgz", + "integrity": "sha512-PuFYYtnQ8IyVl6ib9d3PepeehcUeHN9IO5N/iCRhyg9tStQcqGQBRVHmfmMWPDERU3KwZoHFvbJ4FPXPspvzbA==", + "dev": true, + "requires": { + "@npmcli/ci-detect": "^1.0.0", + "lru-cache": "^6.0.0", + "make-fetch-happen": "^8.0.9", + "minipass": "^3.1.3", + "minipass-fetch": "^1.3.0", + "minipass-json-stream": "^1.0.1", + "minizlib": "^2.0.0", + "npm-package-arg": "^8.0.0" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "socks-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/socks-proxy-agent/-/socks-proxy-agent-5.0.1.tgz", + "integrity": "sha512-vZdmnjb9a2Tz6WEQVIurybSwElwPxMZaIc7PzqbJTrezcKNznv6giT7J7tZDZ1BojVaa1jvO/UiUdhDVB0ACoQ==", + "dev": true, + "requires": { + "agent-base": "^6.0.2", + "debug": "4", + "socks": "^2.3.3" + } + }, + "ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "dev": true, + "requires": { + "minipass": "^3.1.1" + } + }, + "tar": { + "version": "6.1.11", + "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/pulse-till-done": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/pulse-till-done/-/pulse-till-done-4.0.0.tgz", + "integrity": "sha512-Frb4F7QGckaybRhbF7aosLsJ5e9WuH7h0KUkjlzSByVycxY91UZgaEIVjS2oN9wQLrheLMHl6SiFY0/Pvo0Cxg==", + "dev": true, + "requires": { + "npmlog": "^4.1.2" + } + }, + "@lerna/query-graph": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/query-graph/-/query-graph-4.0.0.tgz", + "integrity": "sha512-YlP6yI3tM4WbBmL9GCmNDoeQyzcyg1e4W96y/PKMZa5GbyUvkS2+Jc2kwPD+5KcXou3wQZxSPzR3Te5OenaDdg==", + "dev": true, + "requires": { + "@lerna/package-graph": "4.0.0" + } + }, + "@lerna/resolve-symlink": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/resolve-symlink/-/resolve-symlink-4.0.0.tgz", + "integrity": "sha512-RtX8VEUzqT+uLSCohx8zgmjc6zjyRlh6i/helxtZTMmc4+6O4FS9q5LJas2uGO2wKvBlhcD6siibGt7dIC3xZA==", + "dev": true, + "requires": { + "fs-extra": "^9.1.0", + "npmlog": "^4.1.2", + "read-cmd-shim": "^2.0.0" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + } + } + }, + "@lerna/rimraf-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/rimraf-dir/-/rimraf-dir-4.0.0.tgz", + "integrity": "sha512-QNH9ABWk9mcMJh2/muD9iYWBk1oQd40y6oH+f3wwmVGKYU5YJD//+zMiBI13jxZRtwBx0vmBZzkBkK1dR11cBg==", + "dev": true, + "requires": { + "@lerna/child-process": "4.0.0", + "npmlog": "^4.1.2", + "path-exists": "^4.0.0", + "rimraf": "^3.0.2" + }, + "dependencies": { + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "@lerna/run": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/run/-/run-4.0.0.tgz", + "integrity": "sha512-9giulCOzlMPzcZS/6Eov6pxE9gNTyaXk0Man+iCIdGJNMrCnW7Dme0Z229WWP/UoxDKg71F2tMsVVGDiRd8fFQ==", + "dev": true, + "requires": { + "@lerna/command": "4.0.0", + "@lerna/filter-options": "4.0.0", + "@lerna/npm-run-script": "4.0.0", + "@lerna/output": "4.0.0", + "@lerna/profiler": "4.0.0", + "@lerna/run-topologically": "4.0.0", + "@lerna/timer": "4.0.0", + "@lerna/validation-error": "4.0.0", + "p-map": "^4.0.0" + }, + "dependencies": { + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + } + } + }, + "@lerna/run-lifecycle": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/run-lifecycle/-/run-lifecycle-4.0.0.tgz", + "integrity": "sha512-IwxxsajjCQQEJAeAaxF8QdEixfI7eLKNm4GHhXHrgBu185JcwScFZrj9Bs+PFKxwb+gNLR4iI5rpUdY8Y0UdGQ==", + "dev": true, + "requires": { + "@lerna/npm-conf": "4.0.0", + "npm-lifecycle": "^3.1.5", + "npmlog": "^4.1.2" + } + }, + "@lerna/run-topologically": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/run-topologically/-/run-topologically-4.0.0.tgz", + "integrity": "sha512-EVZw9hGwo+5yp+VL94+NXRYisqgAlj0jWKWtAIynDCpghRxCE5GMO3xrQLmQgqkpUl9ZxQFpICgYv5DW4DksQA==", + "dev": true, + "requires": { + "@lerna/query-graph": "4.0.0", + "p-queue": "^6.6.2" + } + }, + "@lerna/symlink-binary": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/symlink-binary/-/symlink-binary-4.0.0.tgz", + "integrity": "sha512-zualodWC4q1QQc1pkz969hcFeWXOsVYZC5AWVtAPTDfLl+TwM7eG/O6oP+Rr3fFowspxo6b1TQ6sYfDV6HXNWA==", + "dev": true, + "requires": { + "@lerna/create-symlink": "4.0.0", + "@lerna/package": "4.0.0", + "fs-extra": "^9.1.0", + "p-map": "^4.0.0" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + } + } + }, + "@lerna/symlink-dependencies": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/symlink-dependencies/-/symlink-dependencies-4.0.0.tgz", + "integrity": "sha512-BABo0MjeUHNAe2FNGty1eantWp8u83BHSeIMPDxNq0MuW2K3CiQRaeWT3EGPAzXpGt0+hVzBrA6+OT0GPn7Yuw==", + "dev": true, + "requires": { + "@lerna/create-symlink": "4.0.0", + "@lerna/resolve-symlink": "4.0.0", + "@lerna/symlink-binary": "4.0.0", + "fs-extra": "^9.1.0", + "p-map": "^4.0.0", + "p-map-series": "^2.1.0" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + } + } + }, + "@lerna/timer": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/timer/-/timer-4.0.0.tgz", + "integrity": "sha512-WFsnlaE7SdOvjuyd05oKt8Leg3ENHICnvX3uYKKdByA+S3g+TCz38JsNs7OUZVt+ba63nC2nbXDlUnuT2Xbsfg==", + "dev": true + }, + "@lerna/validation-error": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/validation-error/-/validation-error-4.0.0.tgz", + "integrity": "sha512-1rBOM5/koiVWlRi3V6dB863E1YzJS8v41UtsHgMr6gB2ncJ2LsQtMKlJpi3voqcgh41H8UsPXR58RrrpPpufyw==", + "dev": true, + "requires": { + "npmlog": "^4.1.2" + } + }, + "@lerna/version": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/version/-/version-4.0.0.tgz", + "integrity": "sha512-otUgiqs5W9zGWJZSCCMRV/2Zm2A9q9JwSDS7s/tlKq4mWCYriWo7+wsHEA/nPTMDyYyBO5oyZDj+3X50KDUzeA==", + "dev": true, + "requires": { + "@lerna/check-working-tree": "4.0.0", + "@lerna/child-process": "4.0.0", + "@lerna/collect-updates": "4.0.0", + "@lerna/command": "4.0.0", + "@lerna/conventional-commits": "4.0.0", + "@lerna/github-client": "4.0.0", + "@lerna/gitlab-client": "4.0.0", + "@lerna/output": "4.0.0", + "@lerna/prerelease-id-from-version": "4.0.0", + "@lerna/prompt": "4.0.0", + "@lerna/run-lifecycle": "4.0.0", + "@lerna/run-topologically": "4.0.0", + "@lerna/validation-error": "4.0.0", + "chalk": "^4.1.0", + "dedent": "^0.7.0", + "load-json-file": "^6.2.0", + "minimatch": "^3.0.4", + "npmlog": "^4.1.2", + "p-map": "^4.0.0", + "p-pipe": "^3.1.0", + "p-reduce": "^2.1.0", + "p-waterfall": "^2.1.1", + "semver": "^7.3.4", + "slash": "^3.0.0", + "temp-write": "^4.0.0", + "write-json-file": "^4.3.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@lerna/write-log-file": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@lerna/write-log-file/-/write-log-file-4.0.0.tgz", + "integrity": "sha512-XRG5BloiArpXRakcnPHmEHJp+4AtnhRtpDIHSghmXD5EichI1uD73J7FgPp30mm2pDRq3FdqB0NbwSEsJ9xFQg==", + "dev": true, + "requires": { + "npmlog": "^4.1.2", + "write-file-atomic": "^3.0.3" + }, + "dependencies": { + "write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, + "requires": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + } + } + }, + "@mrmlnc/readdir-enhanced": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", + "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", + "dev": true, + "requires": { + "call-me-maybe": "^1.0.1", + "glob-to-regexp": "^0.3.0" + } + }, + "@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "dependencies": { + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true + } + } + }, + "@nodelib/fs.stat": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz", + "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + } + }, + "@npmcli/ci-detect": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/@npmcli/ci-detect/-/ci-detect-1.4.0.tgz", + "integrity": "sha512-3BGrt6FLjqM6br5AhWRKTr3u5GIVkjRYeAFrMp3HjnfICrg4xOrVRwFavKT6tsp++bq5dluL5t8ME/Nha/6c1Q==", + "dev": true + }, + "@npmcli/fs": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/@npmcli/fs/-/fs-1.1.1.tgz", + "integrity": "sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==", + "dev": true, + "requires": { + "@gar/promisify": "^1.0.1", + "semver": "^7.3.5" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@npmcli/git": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/@npmcli/git/-/git-2.1.0.tgz", + "integrity": "sha512-/hBFX/QG1b+N7PZBFs0bi+evgRZcK9nWBxQKZkGoXUT5hJSwl5c4d7y8/hm+NQZRPhQ67RzFaj5UM9YeyKoryw==", + "dev": true, + "requires": { + "@npmcli/promise-spawn": "^1.3.2", + "lru-cache": "^6.0.0", + "mkdirp": "^1.0.4", + "npm-pick-manifest": "^6.1.1", + "promise-inflight": "^1.0.1", + "promise-retry": "^2.0.1", + "semver": "^7.3.5", + "which": "^2.0.2" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@npmcli/installed-package-contents": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@npmcli/installed-package-contents/-/installed-package-contents-1.0.7.tgz", + "integrity": "sha512-9rufe0wnJusCQoLpV9ZPKIVP55itrM5BxOXs10DmdbRfgWtHy1LDyskbwRnBghuB0PrF7pNPOqREVtpz4HqzKw==", + "dev": true, + "requires": { + "npm-bundled": "^1.1.1", + "npm-normalize-package-bin": "^1.0.1" + } + }, + "@npmcli/move-file": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/@npmcli/move-file/-/move-file-1.1.2.tgz", + "integrity": "sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==", + "dev": true, + "requires": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + }, + "dependencies": { + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "@npmcli/node-gyp": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/@npmcli/node-gyp/-/node-gyp-1.0.3.tgz", + "integrity": "sha512-fnkhw+fmX65kiLqk6E3BFLXNC26rUhK90zVwe2yncPliVT/Qos3xjhTLE59Df8KnPlcwIERXKVlU1bXoUQ+liA==", + "dev": true + }, + "@npmcli/promise-spawn": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/@npmcli/promise-spawn/-/promise-spawn-1.3.2.tgz", + "integrity": "sha512-QyAGYo/Fbj4MXeGdJcFzZ+FkDkomfRBrPM+9QYJSg+PxgAUL+LU3FneQk37rKR2/zjqkCV1BLHccX98wRXG3Sg==", + "dev": true, + "requires": { + "infer-owner": "^1.0.4" + } + }, + "@npmcli/run-script": { + "version": "1.8.6", + "resolved": "https://registry.npmmirror.com/@npmcli/run-script/-/run-script-1.8.6.tgz", + "integrity": "sha512-e42bVZnC6VluBZBAFEr3YrdqSspG3bgilyg4nSLBJ7TRGNCzxHa92XAHxQBLYg0BmgwO4b2mf3h/l5EkEWRn3g==", + "dev": true, + "requires": { + "@npmcli/node-gyp": "^1.0.2", + "@npmcli/promise-spawn": "^1.3.2", + "node-gyp": "^7.1.0", + "read-package-json-fast": "^2.0.1" + }, + "dependencies": { + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "node-gyp": { + "version": "7.1.2", + "resolved": "https://registry.npmmirror.com/node-gyp/-/node-gyp-7.1.2.tgz", + "integrity": "sha512-CbpcIo7C3eMu3dL1c3d0xw449fHIGALIJsRP4DDPHpyiW8vcriNY7ubh9TE4zEKfSxscY7PjeFnshE7h75ynjQ==", + "dev": true, + "requires": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.3", + "nopt": "^5.0.0", + "npmlog": "^4.1.2", + "request": "^2.88.2", + "rimraf": "^3.0.2", + "semver": "^7.3.2", + "tar": "^6.0.2", + "which": "^2.0.2" + } + }, + "nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "dev": true, + "requires": { + "abbrev": "1" + } + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "tar": { + "version": "6.1.11", + "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "@octokit/auth-token": { + "version": "2.5.0", + "resolved": "https://registry.npmmirror.com/@octokit/auth-token/-/auth-token-2.5.0.tgz", + "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", + "dev": true, + "requires": { + "@octokit/types": "^6.0.3" + } + }, + "@octokit/core": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/@octokit/core/-/core-3.6.0.tgz", + "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", + "dev": true, + "requires": { + "@octokit/auth-token": "^2.4.4", + "@octokit/graphql": "^4.5.8", + "@octokit/request": "^5.6.3", + "@octokit/request-error": "^2.0.5", + "@octokit/types": "^6.0.3", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/endpoint": { + "version": "6.0.12", + "resolved": "https://registry.npmmirror.com/@octokit/endpoint/-/endpoint-6.0.12.tgz", + "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", + "dev": true, + "requires": { + "@octokit/types": "^6.0.3", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "dev": true + } + } + }, + "@octokit/graphql": { + "version": "4.8.0", + "resolved": "https://registry.npmmirror.com/@octokit/graphql/-/graphql-4.8.0.tgz", + "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", + "dev": true, + "requires": { + "@octokit/request": "^5.6.0", + "@octokit/types": "^6.0.3", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "11.2.0", + "resolved": "https://registry.npmmirror.com/@octokit/openapi-types/-/openapi-types-11.2.0.tgz", + "integrity": "sha512-PBsVO+15KSlGmiI8QAzaqvsNlZlrDlyAJYcrXBCvVUxCp7VnXjkwPoFHgjEJXx3WF9BAwkA6nfCUA7i9sODzKA==", + "dev": true + }, + "@octokit/plugin-enterprise-rest": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz", + "integrity": "sha512-93uGjlhUD+iNg1iWhUENAtJata6w5nE+V4urXOAlIXdco6xNZtUSfYY8dzp3Udy74aqO/B5UZL80x/YMa5PKRw==", + "dev": true + }, + "@octokit/plugin-paginate-rest": { + "version": "2.17.0", + "resolved": "https://registry.npmmirror.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.17.0.tgz", + "integrity": "sha512-tzMbrbnam2Mt4AhuyCHvpRkS0oZ5MvwwcQPYGtMv4tUa5kkzG58SVB0fcsLulOZQeRnOgdkZWkRUiyBlh0Bkyw==", + "dev": true, + "requires": { + "@octokit/types": "^6.34.0" + } + }, + "@octokit/plugin-request-log": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz", + "integrity": "sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA==", + "dev": true + }, + "@octokit/plugin-rest-endpoint-methods": { + "version": "5.13.0", + "resolved": "https://registry.npmmirror.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.13.0.tgz", + "integrity": "sha512-uJjMTkN1KaOIgNtUPMtIXDOjx6dGYysdIFhgA52x4xSadQCz3b/zJexvITDVpANnfKPW/+E0xkOvLntqMYpviA==", + "dev": true, + "requires": { + "@octokit/types": "^6.34.0", + "deprecation": "^2.3.1" + } + }, + "@octokit/request": { + "version": "5.6.3", + "resolved": "https://registry.npmmirror.com/@octokit/request/-/request-5.6.3.tgz", + "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", + "dev": true, + "requires": { + "@octokit/endpoint": "^6.0.1", + "@octokit/request-error": "^2.1.0", + "@octokit/types": "^6.16.1", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.7", + "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "dev": true + } + } + }, + "@octokit/request-error": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/@octokit/request-error/-/request-error-2.1.0.tgz", + "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "dev": true, + "requires": { + "@octokit/types": "^6.0.3", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/rest": { + "version": "18.12.0", + "resolved": "https://registry.npmmirror.com/@octokit/rest/-/rest-18.12.0.tgz", + "integrity": "sha512-gDPiOHlyGavxr72y0guQEhLsemgVjwRePayJ+FcKc2SJqKUbxbkvf5kAZEWA/MKvsfYlQAMVzNJE3ezQcxMJ2Q==", + "dev": true, + "requires": { + "@octokit/core": "^3.5.1", + "@octokit/plugin-paginate-rest": "^2.16.8", + "@octokit/plugin-request-log": "^1.0.4", + "@octokit/plugin-rest-endpoint-methods": "^5.12.0" + } + }, + "@octokit/types": { + "version": "6.34.0", + "resolved": "https://registry.npmmirror.com/@octokit/types/-/types-6.34.0.tgz", + "integrity": "sha512-s1zLBjWhdEI2zwaoSgyOFoKSl109CUcVBCc7biPJ3aAf6LGLU6szDvi31JPU7bxfla2lqfhjbbg/5DdFNxOwHw==", + "dev": true, + "requires": { + "@octokit/openapi-types": "^11.2.0" + } + }, + "@samverschueren/stream-to-observable": { + "version": "0.3.1", + "resolved": "https://registry.npmmirror.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.1.tgz", + "integrity": "sha512-c/qwwcHyafOQuVQJj0IlBjf5yYgBI7YPJ77k4fOJYesb41jio65eaJODRUmfYKhTOFBrIZ66kgvGPlNbjuoRdQ==", + "dev": true, + "requires": { + "any-observable": "^0.3.0" + } + }, + "@soda/friendly-errors-webpack-plugin": { + "version": "1.8.1", + "resolved": "https://registry.npmmirror.com/@soda/friendly-errors-webpack-plugin/-/friendly-errors-webpack-plugin-1.8.1.tgz", + "integrity": "sha512-h2ooWqP8XuFqTXT+NyAFbrArzfQA7R6HTezADrvD9Re8fxMLTPPniLdqVTdDaO0eIoLaAwKT+d6w+5GeTk7Vbg==", + "dev": true, + "requires": { + "chalk": "^3.0.0", + "error-stack-parser": "^2.0.6", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "dev": true + }, + "@types/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/@types/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==", + "dev": true, + "requires": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "@types/json-schema": { + "version": "7.0.11", + "resolved": "https://registry.npmmirror.com/@types/json-schema/-/json-schema-7.0.11.tgz", + "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", + "dev": true + }, + "@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmmirror.com/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", + "dev": true + }, + "@types/minimatch": { + "version": "3.0.5", + "resolved": "https://registry.npmmirror.com/@types/minimatch/-/minimatch-3.0.5.tgz", + "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==", + "dev": true + }, + "@types/minimist": { + "version": "1.2.2", + "resolved": "https://registry.npmmirror.com/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", + "dev": true + }, + "@types/node": { + "version": "17.0.27", + "resolved": "https://registry.npmmirror.com/@types/node/-/node-17.0.27.tgz", + "integrity": "sha512-4/Ke7bbWOasuT3kceBZFGakP1dYN2XFd8v2l9bqF2LNWrmeU07JLpp56aEeG6+Q3olqO5TvXpW0yaiYnZJ5CXg==", + "dev": true + }, + "@types/normalize-package-data": { + "version": "2.4.1", + "resolved": "https://registry.npmmirror.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz", + "integrity": "sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==", + "dev": true + }, + "@types/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@types/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", + "dev": true + }, + "@types/q": { + "version": "1.5.5", + "resolved": "https://registry.npmmirror.com/@types/q/-/q-1.5.5.tgz", + "integrity": "sha512-L28j2FcJfSZOnL1WBjDYp2vUHCeIFlyYI/53EwD/rKUBQ7MtUUfbQWiyKJGpcnv4/WgrhWsFKrcPstcAt/J0tQ==" + }, + "@vue/babel-helper-vue-jsx-merge-props": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/@vue/babel-helper-vue-jsx-merge-props/-/babel-helper-vue-jsx-merge-props-1.2.1.tgz", + "integrity": "sha512-QOi5OW45e2R20VygMSNhyQHvpdUwQZqGPc748JLGCYEy+yp8fNFNdbNIGAgZmi9e+2JHPd6i6idRuqivyicIkA==", + "dev": true + }, + "@vue/babel-plugin-transform-vue-jsx": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/@vue/babel-plugin-transform-vue-jsx/-/babel-plugin-transform-vue-jsx-1.2.1.tgz", + "integrity": "sha512-HJuqwACYehQwh1fNT8f4kyzqlNMpBuUK4rSiSES5D4QsYncv5fxFsLyrxFPG2ksO7t5WP+Vgix6tt6yKClwPzA==", + "dev": true, + "requires": { + "@babel/helper-module-imports": "^7.0.0", + "@babel/plugin-syntax-jsx": "^7.2.0", + "@vue/babel-helper-vue-jsx-merge-props": "^1.2.1", + "html-tags": "^2.0.0", + "lodash.kebabcase": "^4.1.1", + "svg-tags": "^1.0.0" + } + }, + "@vue/babel-preset-app": { + "version": "3.12.1", + "resolved": "https://registry.npmmirror.com/@vue/babel-preset-app/-/babel-preset-app-3.12.1.tgz", + "integrity": "sha512-Zjy5jQaikV1Pz+ri0YgXFS7q4/5wCxB5tRkDOEIt5+4105u0Feb/pvH20nVL6nx9GyXrECFfcm7Yxr/z++OaPQ==", + "dev": true, + "requires": { + "@babel/helper-module-imports": "^7.0.0", + "@babel/plugin-proposal-class-properties": "^7.0.0", + "@babel/plugin-proposal-decorators": "^7.1.0", + "@babel/plugin-syntax-dynamic-import": "^7.0.0", + "@babel/plugin-syntax-jsx": "^7.0.0", + "@babel/plugin-transform-runtime": "^7.4.0", + "@babel/preset-env": "^7.0.0 < 7.4.0", + "@babel/runtime": "^7.0.0", + "@babel/runtime-corejs2": "^7.2.0", + "@vue/babel-preset-jsx": "^1.0.0", + "babel-plugin-dynamic-import-node": "^2.2.0", + "babel-plugin-module-resolver": "3.2.0", + "core-js": "^2.6.5" + } + }, + "@vue/babel-preset-jsx": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/@vue/babel-preset-jsx/-/babel-preset-jsx-1.2.4.tgz", + "integrity": "sha512-oRVnmN2a77bYDJzeGSt92AuHXbkIxbf/XXSE3klINnh9AXBmVS1DGa1f0d+dDYpLfsAKElMnqKTQfKn7obcL4w==", + "dev": true, + "requires": { + "@vue/babel-helper-vue-jsx-merge-props": "^1.2.1", + "@vue/babel-plugin-transform-vue-jsx": "^1.2.1", + "@vue/babel-sugar-composition-api-inject-h": "^1.2.1", + "@vue/babel-sugar-composition-api-render-instance": "^1.2.4", + "@vue/babel-sugar-functional-vue": "^1.2.2", + "@vue/babel-sugar-inject-h": "^1.2.2", + "@vue/babel-sugar-v-model": "^1.2.3", + "@vue/babel-sugar-v-on": "^1.2.3" + } + }, + "@vue/babel-sugar-composition-api-inject-h": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/@vue/babel-sugar-composition-api-inject-h/-/babel-sugar-composition-api-inject-h-1.2.1.tgz", + "integrity": "sha512-4B3L5Z2G+7s+9Bwbf+zPIifkFNcKth7fQwekVbnOA3cr3Pq71q71goWr97sk4/yyzH8phfe5ODVzEjX7HU7ItQ==", + "dev": true, + "requires": { + "@babel/plugin-syntax-jsx": "^7.2.0" + } + }, + "@vue/babel-sugar-composition-api-render-instance": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/@vue/babel-sugar-composition-api-render-instance/-/babel-sugar-composition-api-render-instance-1.2.4.tgz", + "integrity": "sha512-joha4PZznQMsxQYXtR3MnTgCASC9u3zt9KfBxIeuI5g2gscpTsSKRDzWQt4aqNIpx6cv8On7/m6zmmovlNsG7Q==", + "dev": true, + "requires": { + "@babel/plugin-syntax-jsx": "^7.2.0" + } + }, + "@vue/babel-sugar-functional-vue": { + "version": "1.2.2", + "resolved": "https://registry.npmmirror.com/@vue/babel-sugar-functional-vue/-/babel-sugar-functional-vue-1.2.2.tgz", + "integrity": "sha512-JvbgGn1bjCLByIAU1VOoepHQ1vFsroSA/QkzdiSs657V79q6OwEWLCQtQnEXD/rLTA8rRit4rMOhFpbjRFm82w==", + "dev": true, + "requires": { + "@babel/plugin-syntax-jsx": "^7.2.0" + } + }, + "@vue/babel-sugar-inject-h": { + "version": "1.2.2", + "resolved": "https://registry.npmmirror.com/@vue/babel-sugar-inject-h/-/babel-sugar-inject-h-1.2.2.tgz", + "integrity": "sha512-y8vTo00oRkzQTgufeotjCLPAvlhnpSkcHFEp60+LJUwygGcd5Chrpn5480AQp/thrxVm8m2ifAk0LyFel9oCnw==", + "dev": true, + "requires": { + "@babel/plugin-syntax-jsx": "^7.2.0" + } + }, + "@vue/babel-sugar-v-model": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/@vue/babel-sugar-v-model/-/babel-sugar-v-model-1.2.3.tgz", + "integrity": "sha512-A2jxx87mySr/ulAsSSyYE8un6SIH0NWHiLaCWpodPCVOlQVODCaSpiR4+IMsmBr73haG+oeCuSvMOM+ttWUqRQ==", + "dev": true, + "requires": { + "@babel/plugin-syntax-jsx": "^7.2.0", + "@vue/babel-helper-vue-jsx-merge-props": "^1.2.1", + "@vue/babel-plugin-transform-vue-jsx": "^1.2.1", + "camelcase": "^5.0.0", + "html-tags": "^2.0.0", + "svg-tags": "^1.0.0" + } + }, + "@vue/babel-sugar-v-on": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/@vue/babel-sugar-v-on/-/babel-sugar-v-on-1.2.3.tgz", + "integrity": "sha512-kt12VJdz/37D3N3eglBywV8GStKNUhNrsxChXIV+o0MwVXORYuhDTHJRKPgLJRb/EY3vM2aRFQdxJBp9CLikjw==", + "dev": true, + "requires": { + "@babel/plugin-syntax-jsx": "^7.2.0", + "@vue/babel-plugin-transform-vue-jsx": "^1.2.1", + "camelcase": "^5.0.0" + } + }, + "@vue/cli-overlay": { + "version": "3.12.1", + "resolved": "https://registry.npmmirror.com/@vue/cli-overlay/-/cli-overlay-3.12.1.tgz", + "integrity": "sha512-Bym92EN+lj+cNRN2ozbYyH+V8DMXWGbCDUk+hiJ4EYDBZfBkZKvalk1/mOBFwyxiopnnbOEBAAhL/UuMQ1xARg==", + "dev": true + }, + "@vue/cli-plugin-babel": { + "version": "3.12.1", + "resolved": "https://registry.npmmirror.com/@vue/cli-plugin-babel/-/cli-plugin-babel-3.12.1.tgz", + "integrity": "sha512-Zetvz8PikLCGomeKOKu8pC9YQ7cfxs7pGpvEOzaxGdhMnebhjAYR6i6dOB57A6N5lhxQksXCtYTv26QgfiIpdg==", + "dev": true, + "requires": { + "@babel/core": "^7.0.0", + "@vue/babel-preset-app": "^3.12.1", + "@vue/cli-shared-utils": "^3.12.1", + "babel-loader": "^8.0.5", + "webpack": "^4.0.0" + } + }, + "@vue/cli-plugin-eslint": { + "version": "3.12.1", + "resolved": "https://registry.npmmirror.com/@vue/cli-plugin-eslint/-/cli-plugin-eslint-3.12.1.tgz", + "integrity": "sha512-tVTZlEZsy3sQbO4LLWFK11yzlWwqVAqaM+IY+BeWHITBzEJKh2KmouG+x6x/reXiU3qROsMJ4Ej3Hs8buSMWyQ==", + "dev": true, + "requires": { + "@vue/cli-shared-utils": "^3.12.1", + "babel-eslint": "^10.0.1", + "eslint": "^4.19.1", + "eslint-loader": "^2.1.2", + "eslint-plugin-vue": "^4.7.1", + "globby": "^9.2.0", + "webpack": "^4.0.0", + "yorkie": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmmirror.com/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha512-Ajr4IcMXq/2QmMkEmSvxqfLN5zGmJ92gHXAeOXq1OekoH2rfDNsgdDoL2f7QaRCy7G/E6TpxBVdRuNraMztGHw==", + "dev": true, + "optional": true, + "requires": { + "co": "^4.6.0", + "fast-deep-equal": "^1.0.0", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.3.0" + } + }, + "ansi-regex": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", + "dev": true, + "optional": true + }, + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==", + "dev": true, + "optional": true, + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "optional": true, + "requires": { + "ms": "^2.1.1" + } + }, + "eslint": { + "version": "4.19.1", + "resolved": "https://registry.npmmirror.com/eslint/-/eslint-4.19.1.tgz", + "integrity": "sha512-bT3/1x1EbZB7phzYu7vCr1v3ONuzDtX8WjuM9c0iYxe+cq+pwcKEoQjl7zd3RpC6YOLgnSy3cTN58M2jcoPDIQ==", + "dev": true, + "optional": true, + "requires": { + "ajv": "^5.3.0", + "babel-code-frame": "^6.22.0", + "chalk": "^2.1.0", + "concat-stream": "^1.6.0", + "cross-spawn": "^5.1.0", + "debug": "^3.1.0", + "doctrine": "^2.1.0", + "eslint-scope": "^3.7.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^3.5.4", + "esquery": "^1.0.0", + "esutils": "^2.0.2", + "file-entry-cache": "^2.0.0", + "functional-red-black-tree": "^1.0.1", + "glob": "^7.1.2", + "globals": "^11.0.1", + "ignore": "^3.3.3", + "imurmurhash": "^0.1.4", + "inquirer": "^3.0.6", + "is-resolvable": "^1.0.0", + "js-yaml": "^3.9.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.4", + "minimatch": "^3.0.2", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", + "pluralize": "^7.0.0", + "progress": "^2.0.0", + "regexpp": "^1.0.1", + "require-uncached": "^1.0.3", + "semver": "^5.3.0", + "strip-ansi": "^4.0.0", + "strip-json-comments": "~2.0.1", + "table": "4.0.2", + "text-table": "~0.2.0" + } + }, + "eslint-plugin-vue": { + "version": "4.7.1", + "resolved": "https://registry.npmmirror.com/eslint-plugin-vue/-/eslint-plugin-vue-4.7.1.tgz", + "integrity": "sha512-esETKhVMI7Vdli70Wt4bvAwnZBJeM0pxVX9Yb0wWKxdCJc2EADalVYK/q2FzMw8oKN0wPMdqVCKS8kmR89recA==", + "dev": true, + "optional": true, + "requires": { + "vue-eslint-parser": "^2.0.3" + } + }, + "eslint-scope": { + "version": "3.7.3", + "resolved": "https://registry.npmmirror.com/eslint-scope/-/eslint-scope-3.7.3.tgz", + "integrity": "sha512-W+B0SvF4gamyCTmUc+uITPY0989iXVfKvhwtmJocTaYoc/3khEHmEmvfY/Gn9HA9VV75jrQECsHizkNw1b68FA==", + "dev": true, + "optional": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha512-fueX787WZKCV0Is4/T2cyAdM4+x1S3MXXOAhavE1ys/W42SHAPacLTQhucja22QBYrfGw50M2sRiXPtTGv9Ymw==", + "dev": true, + "optional": true + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmmirror.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha512-4JD/Ivzg7PoW8NzdrBSr3UFwC9mHgvI7Z6z3QGBsSHgKaRTUDmyZAAKJo2UbG1kUVfS9WS8bi36N49U1xw43DA==", + "dev": true, + "optional": true + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "optional": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "optional": true + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "dev": true, + "optional": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", + "dev": true, + "optional": true + } + } + }, + "@vue/cli-service": { + "version": "3.12.1", + "resolved": "https://registry.npmmirror.com/@vue/cli-service/-/cli-service-3.12.1.tgz", + "integrity": "sha512-PDxNrTGnSKzeV1ruFlsRIAO8JcPizwT0EJXq9GeyooU+p+sOkv7aKkCBJQVYNjZapD1NOGWx6CvAAC/wAW+gew==", + "dev": true, + "requires": { + "@intervolga/optimize-cssnano-plugin": "^1.0.5", + "@soda/friendly-errors-webpack-plugin": "^1.7.1", + "@vue/cli-overlay": "^3.12.1", + "@vue/cli-shared-utils": "^3.12.1", + "@vue/component-compiler-utils": "^3.0.0", + "@vue/preload-webpack-plugin": "^1.1.0", + "@vue/web-component-wrapper": "^1.2.0", + "acorn": "^6.1.1", + "acorn-walk": "^6.1.1", + "address": "^1.0.3", + "autoprefixer": "^9.5.1", + "browserslist": "^4.5.4", + "cache-loader": "^2.0.1", + "case-sensitive-paths-webpack-plugin": "^2.2.0", + "chalk": "^2.4.2", + "cli-highlight": "^2.1.0", + "clipboardy": "^2.0.0", + "cliui": "^5.0.0", + "copy-webpack-plugin": "^4.6.0", + "css-loader": "^1.0.1", + "cssnano": "^4.1.10", + "current-script-polyfill": "^1.0.0", + "debug": "^4.1.1", + "default-gateway": "^5.0.2", + "dotenv": "^7.0.0", + "dotenv-expand": "^5.1.0", + "escape-string-regexp": "^1.0.5", + "file-loader": "^3.0.1", + "fs-extra": "^7.0.1", + "globby": "^9.2.0", + "hash-sum": "^1.0.2", + "html-webpack-plugin": "^3.2.0", + "launch-editor-middleware": "^2.2.1", + "lodash.defaultsdeep": "^4.6.1", + "lodash.mapvalues": "^4.6.0", + "lodash.transform": "^4.6.0", + "mini-css-extract-plugin": "^0.8.0", + "minimist": "^1.2.0", + "ora": "^3.4.0", + "portfinder": "^1.0.20", + "postcss-loader": "^3.0.0", + "read-pkg": "^5.0.0", + "semver": "^6.0.0", + "slash": "^2.0.0", + "source-map-url": "^0.4.0", + "ssri": "^6.0.1", + "string.prototype.padend": "^3.0.0", + "terser-webpack-plugin": "^1.2.3", + "thread-loader": "^2.1.2", + "url-loader": "^1.1.2", + "vue-loader": "^15.7.0", + "webpack": "^4.0.0", + "webpack-bundle-analyzer": "^3.3.0", + "webpack-chain": "^4.11.0", + "webpack-dev-server": "^3.4.1", + "webpack-merge": "^4.2.1" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "@vue/cli-shared-utils": { + "version": "3.12.1", + "resolved": "https://registry.npmmirror.com/@vue/cli-shared-utils/-/cli-shared-utils-3.12.1.tgz", + "integrity": "sha512-jFblzRFjutGwu5utOKdVlPlsbA1lBUNNQlAThzNqej+JtTKJjnvjlhjKX0Gq0oOny5FjKWhoyfQ74p9h1qE6JQ==", + "dev": true, + "requires": { + "@hapi/joi": "^15.0.1", + "chalk": "^2.4.1", + "execa": "^1.0.0", + "launch-editor": "^2.2.1", + "lru-cache": "^5.1.1", + "node-ipc": "^9.1.1", + "open": "^6.3.0", + "ora": "^3.4.0", + "request": "^2.87.0", + "request-promise-native": "^1.0.7", + "semver": "^6.0.0", + "string.prototype.padstart": "^3.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "@vue/component-compiler-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/@vue/component-compiler-utils/-/component-compiler-utils-3.3.0.tgz", + "integrity": "sha512-97sfH2mYNU+2PzGrmK2haqffDpVASuib9/w2/noxiFi31Z54hW+q3izKQXXQZSNhtiUpAI36uSuYepeBe4wpHQ==", + "dev": true, + "requires": { + "consolidate": "^0.15.1", + "hash-sum": "^1.0.2", + "lru-cache": "^4.1.2", + "merge-source-map": "^1.1.0", + "postcss": "^7.0.36", + "postcss-selector-parser": "^6.0.2", + "prettier": "^1.18.2 || ^2.0.0", + "source-map": "~0.6.1", + "vue-template-es2015-compiler": "^1.9.0" + }, + "dependencies": { + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", + "dev": true + } + } + }, + "@vue/eslint-config-standard": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@vue/eslint-config-standard/-/eslint-config-standard-4.0.0.tgz", + "integrity": "sha512-bQghq1cw1BuMRHNhr3tRpAJx1tpGy0QtajQX873kLtA9YVuOIoXR7nAWnTN09bBHnSUh2N288vMsqPi2fI4Hzg==", + "dev": true, + "requires": { + "eslint-config-standard": "^12.0.0", + "eslint-plugin-import": "^2.14.0", + "eslint-plugin-node": "^8.0.0", + "eslint-plugin-promise": "^4.0.1", + "eslint-plugin-standard": "^4.0.0" + } + }, + "@vue/preload-webpack-plugin": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/@vue/preload-webpack-plugin/-/preload-webpack-plugin-1.1.2.tgz", + "integrity": "sha512-LIZMuJk38pk9U9Ur4YzHjlIyMuxPlACdBIHH9/nGYVTsaGKOSnSuELiE8vS9wa+dJpIYspYUOqk+L1Q4pgHQHQ==", + "dev": true + }, + "@vue/web-component-wrapper": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/@vue/web-component-wrapper/-/web-component-wrapper-1.3.0.tgz", + "integrity": "sha512-Iu8Tbg3f+emIIMmI2ycSI8QcEuAUgPTgHwesDU1eKMLE4YC/c/sFbGc70QgMq31ijRftV0R7vCm9co6rldCeOA==", + "dev": true + }, + "@webassemblyjs/ast": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/ast/-/ast-1.9.0.tgz", + "integrity": "sha512-C6wW5L+b7ogSDVqymbkkvuW9kruN//YisMED04xzeBBqjHa2FYnmvOlS6Xj68xWQRgWvI9cIglsjFowH/RJyEA==", + "requires": { + "@webassemblyjs/helper-module-context": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/wast-parser": "1.9.0" + } + }, + "@webassemblyjs/floating-point-hex-parser": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.9.0.tgz", + "integrity": "sha512-TG5qcFsS8QB4g4MhrxK5TqfdNe7Ey/7YL/xN+36rRjl/BlGE/NcBvJcqsRgCP6Z92mRE+7N50pRIi8SmKUbcQA==" + }, + "@webassemblyjs/helper-api-error": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.9.0.tgz", + "integrity": "sha512-NcMLjoFMXpsASZFxJ5h2HZRcEhDkvnNFOAKneP5RbKRzaWJN36NC4jqQHKwStIhGXu5mUWlUUk7ygdtrO8lbmw==" + }, + "@webassemblyjs/helper-buffer": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.9.0.tgz", + "integrity": "sha512-qZol43oqhq6yBPx7YM3m9Bv7WMV9Eevj6kMi6InKOuZxhw+q9hOkvq5e/PpKSiLfyetpaBnogSbNCfBwyB00CA==" + }, + "@webassemblyjs/helper-code-frame": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.9.0.tgz", + "integrity": "sha512-ERCYdJBkD9Vu4vtjUYe8LZruWuNIToYq/ME22igL+2vj2dQ2OOujIZr3MEFvfEaqKoVqpsFKAGsRdBSBjrIvZA==", + "requires": { + "@webassemblyjs/wast-printer": "1.9.0" + } + }, + "@webassemblyjs/helper-fsm": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-fsm/-/helper-fsm-1.9.0.tgz", + "integrity": "sha512-OPRowhGbshCb5PxJ8LocpdX9Kl0uB4XsAjl6jH/dWKlk/mzsANvhwbiULsaiqT5GZGT9qinTICdj6PLuM5gslw==" + }, + "@webassemblyjs/helper-module-context": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-module-context/-/helper-module-context-1.9.0.tgz", + "integrity": "sha512-MJCW8iGC08tMk2enck1aPW+BE5Cw8/7ph/VGZxwyvGbJwjktKkDK7vy7gAmMDx88D7mhDTCNKAW5tED+gZ0W8g==", + "requires": { + "@webassemblyjs/ast": "1.9.0" + } + }, + "@webassemblyjs/helper-wasm-bytecode": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.9.0.tgz", + "integrity": "sha512-R7FStIzyNcd7xKxCZH5lE0Bqy+hGTwS3LJjuv1ZVxd9O7eHCedSdrId/hMOd20I+v8wDXEn+bjfKDLzTepoaUw==" + }, + "@webassemblyjs/helper-wasm-section": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.9.0.tgz", + "integrity": "sha512-XnMB8l3ek4tvrKUUku+IVaXNHz2YsJyOOmz+MMkZvh8h1uSJpSen6vYnw3IoQ7WwEuAhL8Efjms1ZWjqh2agvw==", + "requires": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-buffer": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/wasm-gen": "1.9.0" + } + }, + "@webassemblyjs/ieee754": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/ieee754/-/ieee754-1.9.0.tgz", + "integrity": "sha512-dcX8JuYU/gvymzIHc9DgxTzUUTLexWwt8uCTWP3otys596io0L5aW02Gb1RjYpx2+0Jus1h4ZFqjla7umFniTg==", + "requires": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "@webassemblyjs/leb128": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/leb128/-/leb128-1.9.0.tgz", + "integrity": "sha512-ENVzM5VwV1ojs9jam6vPys97B/S65YQtv/aanqnU7D8aSoHFX8GyhGg0CMfyKNIHBuAVjy3tlzd5QMMINa7wpw==", + "requires": { + "@xtuc/long": "4.2.2" + } + }, + "@webassemblyjs/utf8": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/utf8/-/utf8-1.9.0.tgz", + "integrity": "sha512-GZbQlWtopBTP0u7cHrEx+73yZKrQoBMpwkGEIqlacljhXCkVM1kMQge/Mf+csMJAjEdSwhOyLAS0AoR3AG5P8w==" + }, + "@webassemblyjs/wasm-edit": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.9.0.tgz", + "integrity": "sha512-FgHzBm80uwz5M8WKnMTn6j/sVbqilPdQXTWraSjBwFXSYGirpkSWE2R9Qvz9tNiTKQvoKILpCuTjBKzOIm0nxw==", + "requires": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-buffer": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/helper-wasm-section": "1.9.0", + "@webassemblyjs/wasm-gen": "1.9.0", + "@webassemblyjs/wasm-opt": "1.9.0", + "@webassemblyjs/wasm-parser": "1.9.0", + "@webassemblyjs/wast-printer": "1.9.0" + } + }, + "@webassemblyjs/wasm-gen": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.9.0.tgz", + "integrity": "sha512-cPE3o44YzOOHvlsb4+E9qSqjc9Qf9Na1OO/BHFy4OI91XDE14MjFN4lTMezzaIWdPqHnsTodGGNP+iRSYfGkjA==", + "requires": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/ieee754": "1.9.0", + "@webassemblyjs/leb128": "1.9.0", + "@webassemblyjs/utf8": "1.9.0" + } + }, + "@webassemblyjs/wasm-opt": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.9.0.tgz", + "integrity": "sha512-Qkjgm6Anhm+OMbIL0iokO7meajkzQD71ioelnfPEj6r4eOFuqm4YC3VBPqXjFyyNwowzbMD+hizmprP/Fwkl2A==", + "requires": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-buffer": "1.9.0", + "@webassemblyjs/wasm-gen": "1.9.0", + "@webassemblyjs/wasm-parser": "1.9.0" + } + }, + "@webassemblyjs/wasm-parser": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.9.0.tgz", + "integrity": "sha512-9+wkMowR2AmdSWQzsPEjFU7njh8HTO5MqO8vjwEHuM+AMHioNqSBONRdr0NQQ3dVQrzp0s8lTcYqzUdb7YgELA==", + "requires": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-api-error": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/ieee754": "1.9.0", + "@webassemblyjs/leb128": "1.9.0", + "@webassemblyjs/utf8": "1.9.0" + } + }, + "@webassemblyjs/wast-parser": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wast-parser/-/wast-parser-1.9.0.tgz", + "integrity": "sha512-qsqSAP3QQ3LyZjNC/0jBJ/ToSxfYJ8kYyuiGvtn/8MK89VrNEfwj7BPQzJVHi0jGTRK2dGdJ5PRqhtjzoww+bw==", + "requires": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/floating-point-hex-parser": "1.9.0", + "@webassemblyjs/helper-api-error": "1.9.0", + "@webassemblyjs/helper-code-frame": "1.9.0", + "@webassemblyjs/helper-fsm": "1.9.0", + "@xtuc/long": "4.2.2" + } + }, + "@webassemblyjs/wast-printer": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wast-printer/-/wast-printer-1.9.0.tgz", + "integrity": "sha512-2J0nE95rHXHyQ24cWjMKJ1tqB/ds8z/cyeOZxJhcb+rW+SQASVjuznUSmdz5GpVJTzU8JkhYut0D3siFDD6wsA==", + "requires": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/wast-parser": "1.9.0", + "@xtuc/long": "4.2.2" + } + }, + "@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==" + }, + "@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmmirror.com/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==" + }, + "@yarnpkg/lockfile": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", + "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==", + "dev": true + }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmmirror.com/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "dev": true, + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmmirror.com/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, + "acorn": { + "version": "6.4.2", + "resolved": "https://registry.npmmirror.com/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==" + }, + "acorn-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/acorn-jsx/-/acorn-jsx-3.0.1.tgz", + "integrity": "sha512-AU7pnZkguthwBjKgCg6998ByQNIMjbuDQZ8bb78QAFZwPfmKia8AIzgY/gWgqCjnht8JLdXmB4YxA0KaV60ncQ==", + "dev": true, + "optional": true, + "requires": { + "acorn": "^3.0.4" + }, + "dependencies": { + "acorn": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/acorn/-/acorn-3.3.0.tgz", + "integrity": "sha512-OLUyIIZ7mF5oaAUT1w0TFqQS81q3saT46x8t7ukpPjMNk+nbs4ZHhs7ToV8EWnLYLepjETXd4XaCE4uxkMeqUw==", + "dev": true, + "optional": true + } + } + }, + "acorn-walk": { + "version": "6.2.0", + "resolved": "https://registry.npmmirror.com/acorn-walk/-/acorn-walk-6.2.0.tgz", + "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==", + "dev": true + }, + "add-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/add-stream/-/add-stream-1.0.0.tgz", + "integrity": "sha512-qQLMr+8o0WC4FZGQTcJiKBVC59JylcPSrTtk6usvmIDFUOCKegapy1VHQwRbFMOFyb/inzUVqHs+eMYKDM1YeQ==", + "dev": true + }, + "address": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/address/-/address-1.1.2.tgz", + "integrity": "sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA==", + "dev": true + }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmmirror.com/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "requires": { + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "agentkeepalive": { + "version": "4.2.1", + "resolved": "https://registry.npmmirror.com/agentkeepalive/-/agentkeepalive-4.2.1.tgz", + "integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==", + "dev": true, + "requires": { + "debug": "^4.1.0", + "depd": "^1.1.2", + "humanize-ms": "^1.2.1" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "dev": true, + "requires": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + } + }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmmirror.com/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ajv-errors": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/ajv-errors/-/ajv-errors-1.0.1.tgz", + "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==" + }, + "ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmmirror.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==" + }, + "alphanum-sort": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/alphanum-sort/-/alphanum-sort-1.0.2.tgz", + "integrity": "sha512-0FcBfdcmaumGPQ0qPn7Q5qTgz/ooXgIyp1rf8ik5bGX8mpE2YHjC0P/eyQvxu1GURYQgq9ozf2mteQ5ZD9YiyQ==", + "dev": true + }, + "amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==", + "dev": true + }, + "ansi-colors": { + "version": "3.2.4", + "resolved": "https://registry.npmmirror.com/ansi-colors/-/ansi-colors-3.2.4.tgz", + "integrity": "sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA==", + "dev": true + }, + "ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true + }, + "ansi-html-community": { + "version": "0.0.8", + "resolved": "https://registry.npmmirror.com/ansi-html-community/-/ansi-html-community-0.0.8.tgz", + "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==", + "dev": true + }, + "ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "dev": true + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "any-observable": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/any-observable/-/any-observable-0.3.0.tgz", + "integrity": "sha512-/FQM1EDkTsf63Ub2C6O7GuYFDsSXUwsaZDurV0np41ocwq0jthUAYCmhBX9f+KwlaCgIuWyr/4WlUQUBfKfZog==", + "dev": true + }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true + }, + "anymatch": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-3.1.2.tgz", + "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", + "optional": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "arch": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/arch/-/arch-2.2.0.tgz", + "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", + "dev": true + }, + "archiver": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/archiver/-/archiver-3.1.1.tgz", + "integrity": "sha512-5Hxxcig7gw5Jod/8Gq0OneVgLYET+oNHcxgWItq4TbhOzRLKNAFUb9edAftiMKXvXfCB0vbGrJdZDNq0dWMsxg==", + "dev": true, + "requires": { + "archiver-utils": "^2.1.0", + "async": "^2.6.3", + "buffer-crc32": "^0.2.1", + "glob": "^7.1.4", + "readable-stream": "^3.4.0", + "tar-stream": "^2.1.0", + "zip-stream": "^2.1.2" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "dev": true, + "requires": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + } + }, + "are-we-there-yet": { + "version": "1.1.7", + "resolved": "https://registry.npmmirror.com/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz", + "integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==", + "dev": true, + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmmirror.com/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==" + }, + "arr-flatten": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/arr-flatten/-/arr-flatten-1.1.0.tgz", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==" + }, + "arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==" + }, + "array-differ": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/array-differ/-/array-differ-3.0.0.tgz", + "integrity": "sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg==", + "dev": true + }, + "array-find-index": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/array-find-index/-/array-find-index-1.0.2.tgz", + "integrity": "sha512-M1HQyIXcBGtVywBt8WVdim+lrNaK7VHp99Qt5pSNziXznKHViIBbXWtfRTpEFpF/c4FdfxNAsCCwPp5phBYJtw==", + "dev": true + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true + }, + "array-ify": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/array-ify/-/array-ify-1.0.0.tgz", + "integrity": "sha512-c5AMf34bKdvPhQ7tBGhqkgKNUzMr4WUs+WDtC2ZUGOUncbxKMTvqxYctiseW3+L4bA8ec+GcZ6/A/FW4m8ukng==", + "dev": true + }, + "array-includes": { + "version": "3.1.4", + "resolved": "https://registry.npmmirror.com/array-includes/-/array-includes-3.1.4.tgz", + "integrity": "sha512-ZTNSQkmWumEbiHO2GF4GmWxYVTiQyJy2XOTa15sdQSrvKn7l+180egQMqlrMOUMCyLMD7pmyQe4mMDUT6Behrw==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1", + "get-intrinsic": "^1.1.1", + "is-string": "^1.0.7" + } + }, + "array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha512-Dxr6QJj/RdU/hCaBjOfxW+q6lyuVE6JFWIrAUpuOOhoJJoQ99cUn3igRaHVB5P9WrgFVN0FfArM3x0cueOU8ng==", + "dev": true, + "requires": { + "array-uniq": "^1.0.1" + } + }, + "array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q==", + "dev": true + }, + "array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmmirror.com/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==" + }, + "array.prototype.flat": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/array.prototype.flat/-/array.prototype.flat-1.3.0.tgz", + "integrity": "sha512-12IUEkHsAhA4DY5s0FPgNXIdc8VRSqD9Zp78a5au9abH/SOBrsp082JOWFNTjkMozh8mqcdiKuaLGhPeYztxSw==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.2", + "es-shim-unscopables": "^1.0.0" + } + }, + "arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "dev": true + }, + "asap": { + "version": "2.0.6", + "resolved": "https://registry.npmmirror.com/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "dev": true + }, + "asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmmirror.com/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dev": true, + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmmirror.com/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "requires": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmmirror.com/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "assert": { + "version": "1.5.0", + "resolved": "https://registry.npmmirror.com/assert/-/assert-1.5.0.tgz", + "integrity": "sha512-EDsgawzwoun2CZkCgtxJbv392v4nbk9XDD06zI+kQYoBM/3RBWLlEyJARDOmhAAosBjWACEkKL6S+lIZtcAubA==", + "requires": { + "object-assign": "^4.1.1", + "util": "0.10.3" + }, + "dependencies": { + "inherits": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.1.tgz", + "integrity": "sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==" + }, + "util": { + "version": "0.10.3", + "resolved": "https://registry.npmmirror.com/util/-/util-0.10.3.tgz", + "integrity": "sha512-5KiHfsmkqacuKjkRkdV7SsfDJ2EGiPsK92s2MhNSY0craxjTdKTtqKsJaCWp4LW33ZZ0OPUv1WO/TFvNQRiQxQ==", + "requires": { + "inherits": "2.0.1" + } + } + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==", + "dev": true + }, + "assign-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/assign-symbols/-/assign-symbols-1.0.0.tgz", + "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==" + }, + "astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "dev": true + }, + "async": { + "version": "2.6.4", + "resolved": "https://registry.npmmirror.com/async/-/async-2.6.4.tgz", + "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", + "dev": true, + "requires": { + "lodash": "^4.17.14" + } + }, + "async-each": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/async-each/-/async-each-1.0.3.tgz", + "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==" + }, + "async-foreach": { + "version": "0.1.3", + "resolved": "https://registry.npmmirror.com/async-foreach/-/async-foreach-0.1.3.tgz", + "integrity": "sha512-VUeSMD8nEGBWaZK4lizI1sf3yEC7pnAQ/mrI7pC2fBz2s/tq5jWWEngTwaf0Gruu/OoXRGLGg1XFqpYBiGTYJA==", + "dev": true + }, + "async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==", + "dev": true + }, + "async-validator": { + "version": "1.12.2", + "resolved": "https://registry.npmmirror.com/async-validator/-/async-validator-1.12.2.tgz", + "integrity": "sha512-57EETfCPFiB7M4QscvQzWSGNsmtkjjzZv318SK1CBlstk+hycV72ocjriMOOM48HjvmoAoJGpJNjC7Z76RlnZA==" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true + }, + "at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", + "dev": true + }, + "atob": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" + }, + "autoprefixer": { + "version": "9.8.8", + "resolved": "https://registry.npmmirror.com/autoprefixer/-/autoprefixer-9.8.8.tgz", + "integrity": "sha512-eM9d/swFopRt5gdJ7jrpCwgvEMIayITpojhkkSMRsFHYuH5bkSQ4p/9qTEHtmNudUZh22Tehu7I6CxAW0IXTKA==", + "dev": true, + "requires": { + "browserslist": "^4.12.0", + "caniuse-lite": "^1.0.30001109", + "normalize-range": "^0.1.2", + "num2fraction": "^1.2.2", + "picocolors": "^0.2.1", + "postcss": "^7.0.32", + "postcss-value-parser": "^4.1.0" + }, + "dependencies": { + "picocolors": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/picocolors/-/picocolors-0.2.1.tgz", + "integrity": "sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA==", + "dev": true + } + } + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmmirror.com/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==", + "dev": true + }, + "aws4": { + "version": "1.11.0", + "resolved": "https://registry.npmmirror.com/aws4/-/aws4-1.11.0.tgz", + "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==", + "dev": true + }, + "axios": { + "version": "0.21.1", + "resolved": "https://registry.npmmirror.com/axios/-/axios-0.21.1.tgz", + "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, + "babel-code-frame": { + "version": "6.26.0", + "resolved": "https://registry.npmmirror.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz", + "integrity": "sha512-XqYMR2dfdGMW+hd0IUZ2PwK+fGeFkOxZJ0wY+JaQAHzt1Zx8LcvpiZD2NiGkEG8qx0CfkAOr5xt76d1e8vG90g==", + "dev": true, + "requires": { + "chalk": "^1.1.3", + "esutils": "^2.0.2", + "js-tokens": "^3.0.2" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "js-tokens": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/js-tokens/-/js-tokens-3.0.2.tgz", + "integrity": "sha512-RjTcuD4xjtthQkaWH7dFlH85L+QaVtSoOyGdZ3g6HFhS9dFNDfLyqgm2NFe2X6cQpeFmt0452FJjFG5UameExg==", + "dev": true + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true + } + } + }, + "babel-eslint": { + "version": "10.1.0", + "resolved": "https://registry.npmmirror.com/babel-eslint/-/babel-eslint-10.1.0.tgz", + "integrity": "sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/parser": "^7.7.0", + "@babel/traverse": "^7.7.0", + "@babel/types": "^7.7.0", + "eslint-visitor-keys": "^1.0.0", + "resolve": "^1.12.0" + } + }, + "babel-loader": { + "version": "8.2.5", + "resolved": "https://registry.npmmirror.com/babel-loader/-/babel-loader-8.2.5.tgz", + "integrity": "sha512-OSiFfH89LrEMiWd4pLNqGz4CwJDtbs2ZVc+iGu2HrkRfPxId9F2anQj38IxWpmRfsUY0aBZYi1EFcd3mhtRMLQ==", + "dev": true, + "requires": { + "find-cache-dir": "^3.3.1", + "loader-utils": "^2.0.0", + "make-dir": "^3.1.0", + "schema-utils": "^2.6.5" + }, + "dependencies": { + "find-cache-dir": { + "version": "3.3.2", + "resolved": "https://registry.npmmirror.com/find-cache-dir/-/find-cache-dir-3.3.2.tgz", + "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true + }, + "loader-utils": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/loader-utils/-/loader-utils-2.0.2.tgz", + "integrity": "sha512-TM57VeHptv569d/GKh6TAYdzKblwDNiumOdkFnejjD0XwTH87K90w3O7AiJRqdQoXygvi1VQTJTLGhJl7WqA7A==", + "dev": true, + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "requires": { + "semver": "^6.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "requires": { + "find-up": "^4.0.0" + } + }, + "schema-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmmirror.com/schema-utils/-/schema-utils-2.7.1.tgz", + "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.5", + "ajv": "^6.12.4", + "ajv-keywords": "^3.5.2" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmmirror.com/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "dev": true, + "requires": { + "object.assign": "^4.1.0" + } + }, + "babel-plugin-module-resolver": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/babel-plugin-module-resolver/-/babel-plugin-module-resolver-3.2.0.tgz", + "integrity": "sha512-tjR0GvSndzPew/Iayf4uICWZqjBwnlMWjSx6brryfQ81F9rxBVqwDJtFCV8oOs0+vJeefK9TmdZtkIFdFe1UnA==", + "dev": true, + "requires": { + "find-babel-config": "^1.1.0", + "glob": "^7.1.2", + "pkg-up": "^2.0.0", + "reselect": "^3.0.1", + "resolve": "^1.4.0" + } + }, + "babel-plugin-polyfill-corejs2": { + "version": "0.3.1", + "resolved": "https://registry.npmmirror.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.1.tgz", + "integrity": "sha512-v7/T6EQcNfVLfcN2X8Lulb7DjprieyLWJK/zOWH5DUYcAgex9sP3h25Q+DLsX9TloXe3y1O8l2q2Jv9q8UVB9w==", + "dev": true, + "requires": { + "@babel/compat-data": "^7.13.11", + "@babel/helper-define-polyfill-provider": "^0.3.1", + "semver": "^6.1.1" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "babel-plugin-polyfill-corejs3": { + "version": "0.5.2", + "resolved": "https://registry.npmmirror.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.5.2.tgz", + "integrity": "sha512-G3uJih0XWiID451fpeFaYGVuxHEjzKTHtc9uGFEjR6hHrvNzeS/PX+LLLcetJcytsB5m4j+K3o/EpXJNb/5IEQ==", + "dev": true, + "requires": { + "@babel/helper-define-polyfill-provider": "^0.3.1", + "core-js-compat": "^3.21.0" + } + }, + "babel-plugin-polyfill-regenerator": { + "version": "0.3.1", + "resolved": "https://registry.npmmirror.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.3.1.tgz", + "integrity": "sha512-Y2B06tvgHYt1x0yz17jGkGeeMr5FeKUu+ASJ+N6nB5lQ8Dapfg42i0OVrf8PNGJ3zKL4A23snMi1IRwrqqND7A==", + "dev": true, + "requires": { + "@babel/helper-define-polyfill-provider": "^0.3.1" + } + }, + "babel-polyfill": { + "version": "6.26.0", + "resolved": "https://registry.npmmirror.com/babel-polyfill/-/babel-polyfill-6.26.0.tgz", + "integrity": "sha512-F2rZGQnAdaHWQ8YAoeRbukc7HS9QgdgeyJ0rQDd485v9opwuPvjpPFcOOT/WmkKTdgy9ESgSPXDcTNpzrGr6iQ==", + "requires": { + "babel-runtime": "^6.26.0", + "core-js": "^2.5.0", + "regenerator-runtime": "^0.10.5" + }, + "dependencies": { + "regenerator-runtime": { + "version": "0.10.5", + "resolved": "https://registry.npmmirror.com/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz", + "integrity": "sha512-02YopEIhAgiBHWeoTiA8aitHDt8z6w+rQqNuIftlM+ZtvSl/brTouaU7DW6GO/cHtvxJvS4Hwv2ibKdxIRi24w==" + } + } + }, + "babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmmirror.com/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha512-ITKNuq2wKlW1fJg9sSW52eepoYgZBggvOAHC0u/CYu/qxQ9EVzThCgR69BnSXLHjy2f7SY5zaQ4yt7H9ZVxY2g==", + "requires": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + }, + "dependencies": { + "regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmmirror.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + } + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "base": { + "version": "0.11.2", + "resolved": "https://registry.npmmirror.com/base/-/base-0.11.2.tgz", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "requires": { + "cache-base": "^1.0.1", + "class-utils": "^0.3.5", + "component-emitter": "^1.2.1", + "define-property": "^1.0.0", + "isobject": "^3.0.1", + "mixin-deep": "^1.2.0", + "pascalcase": "^0.1.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "requires": { + "is-descriptor": "^1.0.0" + } + }, + "is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "requires": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + } + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmmirror.com/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "batch": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/batch/-/batch-0.6.1.tgz", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", + "dev": true + }, + "batch-processor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/batch-processor/-/batch-processor-1.0.0.tgz", + "integrity": "sha512-xoLQD8gmmR32MeuBHgH0Tzd5PuSZx71ZsbhVxOCRbgktZEPe4SQy7s9Z50uPp0F/f7iw2XmkHN2xkgbMfckMDA==" + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dev": true, + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "before-after-hook": { + "version": "2.2.2", + "resolved": "https://registry.npmmirror.com/before-after-hook/-/before-after-hook-2.2.2.tgz", + "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==", + "dev": true + }, + "bfj": { + "version": "6.1.2", + "resolved": "https://registry.npmmirror.com/bfj/-/bfj-6.1.2.tgz", + "integrity": "sha512-BmBJa4Lip6BPRINSZ0BPEIfB1wUY/9rwbwvIHQA1KjX9om29B6id0wnWXq7m3bn5JrUVjeOTnVuhPT1FiHwPGw==", + "dev": true, + "requires": { + "bluebird": "^3.5.5", + "check-types": "^8.0.3", + "hoopy": "^0.1.4", + "tryer": "^1.0.1" + } + }, + "big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmmirror.com/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==" + }, + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "optional": true + }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmmirror.com/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "block-stream": { + "version": "0.0.9", + "resolved": "https://registry.npmmirror.com/block-stream/-/block-stream-0.0.9.tgz", + "integrity": "sha512-OorbnJVPII4DuUKbjARAe8u8EfqOmkEEaSFIyoQ7OjTHn6kafxWl0wLgoZ2rXaYd7MyLcDaU4TmhfxtwgcccMQ==", + "dev": true, + "requires": { + "inherits": "~2.0.0" + } + }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmmirror.com/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "bn.js": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/bn.js/-/bn.js-5.2.0.tgz", + "integrity": "sha512-D7iWRBvnZE8ecXiLj/9wbxH7Tk79fAh8IHaTNq1RWRixsS02W+5qS+iE9yq6RYl0asXx5tw0bLhmT5pIfbSquw==" + }, + "body-parser": { + "version": "1.20.0", + "resolved": "https://registry.npmmirror.com/body-parser/-/body-parser-1.20.0.tgz", + "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "dev": true, + "requires": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.10.3", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "dependencies": { + "qs": { + "version": "6.10.3", + "resolved": "https://registry.npmmirror.com/qs/-/qs-6.10.3.tgz", + "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "dev": true, + "requires": { + "side-channel": "^1.0.4" + } + } + } + }, + "bonjour": { + "version": "3.5.0", + "resolved": "https://registry.npmmirror.com/bonjour/-/bonjour-3.5.0.tgz", + "integrity": "sha512-RaVTblr+OnEli0r/ud8InrU7D+G0y6aJhlxaLa6Pwty4+xoxboF1BsUI45tujvRpbj9dQVoglChqonGAsjEBYg==", + "dev": true, + "requires": { + "array-flatten": "^2.1.0", + "deep-equal": "^1.0.1", + "dns-equal": "^1.0.0", + "dns-txt": "^2.0.2", + "multicast-dns": "^6.0.1", + "multicast-dns-service-types": "^1.1.0" + }, + "dependencies": { + "array-flatten": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/array-flatten/-/array-flatten-2.1.2.tgz", + "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==", + "dev": true + } + } + }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "2.3.2", + "resolved": "https://registry.npmmirror.com/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "requires": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" + }, + "browserify-aes": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/browserify-aes/-/browserify-aes-1.2.0.tgz", + "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", + "requires": { + "buffer-xor": "^1.0.3", + "cipher-base": "^1.0.0", + "create-hash": "^1.1.0", + "evp_bytestokey": "^1.0.3", + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "browserify-cipher": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/browserify-cipher/-/browserify-cipher-1.0.1.tgz", + "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", + "requires": { + "browserify-aes": "^1.0.4", + "browserify-des": "^1.0.0", + "evp_bytestokey": "^1.0.0" + } + }, + "browserify-des": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/browserify-des/-/browserify-des-1.0.2.tgz", + "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", + "requires": { + "cipher-base": "^1.0.1", + "des.js": "^1.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "browserify-rsa": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/browserify-rsa/-/browserify-rsa-4.1.0.tgz", + "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", + "requires": { + "bn.js": "^5.0.0", + "randombytes": "^2.0.1" + } + }, + "browserify-sign": { + "version": "4.2.1", + "resolved": "https://registry.npmmirror.com/browserify-sign/-/browserify-sign-4.2.1.tgz", + "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "requires": { + "bn.js": "^5.1.1", + "browserify-rsa": "^4.0.1", + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "elliptic": "^6.5.3", + "inherits": "^2.0.4", + "parse-asn1": "^5.1.5", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } + } + }, + "browserify-zlib": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/browserify-zlib/-/browserify-zlib-0.2.0.tgz", + "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "requires": { + "pako": "~1.0.5" + } + }, + "browserslist": { + "version": "4.20.3", + "resolved": "https://registry.npmmirror.com/browserslist/-/browserslist-4.20.3.tgz", + "integrity": "sha512-NBhymBQl1zM0Y5dQT/O+xiLP9/rzOIQdKM/eMJBAq7yBgaB6krIYLGejrwVYnSHZdqjscB1SPuAjHwxjvN6Wdg==", + "dev": true, + "requires": { + "caniuse-lite": "^1.0.30001332", + "electron-to-chromium": "^1.4.118", + "escalade": "^3.1.1", + "node-releases": "^2.0.3", + "picocolors": "^1.0.0" + } + }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmmirror.com/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmmirror.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "dev": true + }, + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "buffer-indexof": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/buffer-indexof/-/buffer-indexof-1.1.1.tgz", + "integrity": "sha512-4/rOEg86jivtPTeOUUT61jJO1Ya1TrR/OkqCSZDyq84WJh3LuuiphBYJN+fm5xufIk4XAFcEwte/8WzC8If/1g==", + "dev": true + }, + "buffer-xor": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/buffer-xor/-/buffer-xor-1.0.3.tgz", + "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==" + }, + "builtin-status-codes": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", + "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==" + }, + "builtins": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/builtins/-/builtins-1.0.3.tgz", + "integrity": "sha512-uYBjakWipfaO/bXI7E8rq6kpwHRZK5cNYrUv2OzZSI/FvmdMyXJ2tG9dKcjEC5YHmHpUAwsargWIZNWdxb/bnQ==", + "dev": true + }, + "butterfly-dag": { + "version": "4.1.23", + "resolved": "https://registry.npmmirror.com/butterfly-dag/-/butterfly-dag-4.1.23.tgz", + "integrity": "sha512-8B7HI5sexTQx9Wexy7pOSVwyMxzKmQxnmz6S7gWzzRuP6O/8Luny4edzkUjnnREE5PBplxK+c6PSJKrny901mQ==", + "requires": { + "@antv/hierarchy": "~0.6.4", + "@antv/matrix-util": "^3.0.4", + "d3-force": "~2.1.1", + "dagre": "~0.8.5", + "dom-to-image": "~2.6.0", + "eventemitter3": "4.0.7", + "ml-matrix": "^6.5.0" + } + }, + "byline": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/byline/-/byline-5.0.0.tgz", + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==", + "dev": true + }, + "byte-size": { + "version": "7.0.1", + "resolved": "https://registry.npmmirror.com/byte-size/-/byte-size-7.0.1.tgz", + "integrity": "sha512-crQdqyCwhokxwV1UyDzLZanhkugAgft7vt0qbbdt60C6Zf3CAiGmtUCylbtYwrU6loOUw3euGrNtW1J651ot1A==", + "dev": true + }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true + }, + "cacache": { + "version": "12.0.4", + "resolved": "https://registry.npmmirror.com/cacache/-/cacache-12.0.4.tgz", + "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==", + "requires": { + "bluebird": "^3.5.5", + "chownr": "^1.1.1", + "figgy-pudding": "^3.5.1", + "glob": "^7.1.4", + "graceful-fs": "^4.1.15", + "infer-owner": "^1.0.3", + "lru-cache": "^5.1.1", + "mississippi": "^3.0.0", + "mkdirp": "^0.5.1", + "move-concurrently": "^1.0.1", + "promise-inflight": "^1.0.1", + "rimraf": "^2.6.3", + "ssri": "^6.0.1", + "unique-filename": "^1.1.1", + "y18n": "^4.0.0" + } + }, + "cache-base": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/cache-base/-/cache-base-1.0.1.tgz", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "requires": { + "collection-visit": "^1.0.0", + "component-emitter": "^1.2.1", + "get-value": "^2.0.6", + "has-value": "^1.0.0", + "isobject": "^3.0.1", + "set-value": "^2.0.0", + "to-object-path": "^0.3.0", + "union-value": "^1.0.0", + "unset-value": "^1.0.0" + } + }, + "cache-loader": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/cache-loader/-/cache-loader-2.0.1.tgz", + "integrity": "sha512-V99T3FOynmGx26Zom+JrVBytLBsmUCzVG2/4NnUKgvXN4bEV42R1ERl1IyiH/cvFIDA1Ytq2lPZ9tXDSahcQpQ==", + "dev": true, + "requires": { + "loader-utils": "^1.1.0", + "mkdirp": "^0.5.1", + "neo-async": "^2.6.0", + "normalize-path": "^3.0.0", + "schema-utils": "^1.0.0" + } + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "call-me-maybe": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/call-me-maybe/-/call-me-maybe-1.0.1.tgz", + "integrity": "sha512-wCyFsDQkKPwwF8BDwOiWNx/9K45L/hvggQiDbve+viMNMQnWhrlYIuBk09offfwCRtCO9P6XwUttufzU11WCVw==", + "dev": true + }, + "caller-callsite": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/caller-callsite/-/caller-callsite-2.0.0.tgz", + "integrity": "sha512-JuG3qI4QOftFsZyOn1qq87fq5grLIyk1JYd5lJmdA+fG7aQ9pA/i3JIJGcO3q0MrRcHlOt1U+ZeHW8Dq9axALQ==", + "dev": true, + "requires": { + "callsites": "^2.0.0" + }, + "dependencies": { + "callsites": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/callsites/-/callsites-2.0.0.tgz", + "integrity": "sha512-ksWePWBloaWPxJYQ8TL0JHvtci6G5QTKwQ95RcWAa/lzoAKuAOflGdAK92hpHXjkwb8zLxoLNUoNYZgVsaJzvQ==", + "dev": true + } + } + }, + "caller-path": { + "version": "0.1.0", + "resolved": "https://registry.npmmirror.com/caller-path/-/caller-path-0.1.0.tgz", + "integrity": "sha512-UJiE1otjXPF5/x+T3zTnSFiTOEmJoGTD9HmBoxnCUwho61a2eSNn/VwtwuIBDAo2SEOv1AJ7ARI5gCmohFLu/g==", + "dev": true, + "optional": true, + "requires": { + "callsites": "^0.2.0" + } + }, + "callsites": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/callsites/-/callsites-0.2.0.tgz", + "integrity": "sha512-Zv4Dns9IbXXmPkgRRUjAaJQgfN4xX5p6+RQFhWUqscdvvK2xK/ZL8b3IXIJsj+4sD+f24NwnWy2BY8AJ82JB0A==", + "dev": true, + "optional": true + }, + "camel-case": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/camel-case/-/camel-case-3.0.0.tgz", + "integrity": "sha512-+MbKztAYHXPr1jNTSKQF52VpcFjwY5RkR7fxksV8Doo4KAYc5Fl4UJRgthBbTmEx8C54DqahhbLJkDwjI3PI/w==", + "dev": true, + "requires": { + "no-case": "^2.2.0", + "upper-case": "^1.1.1" + } + }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmmirror.com/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, + "camelcase-keys": { + "version": "6.2.2", + "resolved": "https://registry.npmmirror.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz", + "integrity": "sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==", + "dev": true, + "requires": { + "camelcase": "^5.3.1", + "map-obj": "^4.0.0", + "quick-lru": "^4.0.1" + } + }, + "caniuse-api": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/caniuse-api/-/caniuse-api-3.0.0.tgz", + "integrity": "sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==", + "dev": true, + "requires": { + "browserslist": "^4.0.0", + "caniuse-lite": "^1.0.0", + "lodash.memoize": "^4.1.2", + "lodash.uniq": "^4.5.0" + } + }, + "caniuse-lite": { + "version": "1.0.30001332", + "resolved": "https://registry.npmmirror.com/caniuse-lite/-/caniuse-lite-1.0.30001332.tgz", + "integrity": "sha512-10T30NYOEQtN6C11YGg411yebhvpnC6Z102+B95eAsN0oB6KUs01ivE8u+G6FMIRtIrVlYXhL+LUwQ3/hXwDWw==", + "dev": true + }, + "case-sensitive-paths-webpack-plugin": { + "version": "2.4.0", + "resolved": "https://registry.npmmirror.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz", + "integrity": "sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw==", + "dev": true + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmmirror.com/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", + "dev": true + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "charcodes": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/charcodes/-/charcodes-0.2.0.tgz", + "integrity": "sha512-Y4kiDb+AM4Ecy58YkuZrrSRJBDQdQ2L+NyS1vHHFtNtUjgutcZfx3yp1dAONI/oPaPmyGfCLx5CxL+zauIMyKQ==", + "dev": true + }, + "chardet": { + "version": "0.4.2", + "resolved": "https://registry.npmmirror.com/chardet/-/chardet-0.4.2.tgz", + "integrity": "sha512-j/Toj7f1z98Hh2cYo2BVr85EpIRWqUi7rtRSGxh/cqUjqrnJe9l9UE7IUGd2vQ2p+kSHLkSzObQPZPLUC6TQwg==", + "dev": true, + "optional": true + }, + "charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmmirror.com/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==" + }, + "check-types": { + "version": "8.0.3", + "resolved": "https://registry.npmmirror.com/check-types/-/check-types-8.0.3.tgz", + "integrity": "sha512-YpeKZngUmG65rLudJ4taU7VLkOCTMhNl/u4ctNC56LQS/zJTyNH0Lrtwm1tfTsbLlwvlfsA2d1c8vCf/Kh2KwQ==", + "dev": true + }, + "cheerio": { + "version": "1.0.0-rc.10", + "resolved": "https://registry.npmmirror.com/cheerio/-/cheerio-1.0.0-rc.10.tgz", + "integrity": "sha512-g0J0q/O6mW8z5zxQ3A8E8J1hUgp4SMOvEoW/x84OwyHKe/Zccz83PVT4y5Crcr530FV6NgmKI1qvGTKVl9XXVw==", + "dev": true, + "requires": { + "cheerio-select": "^1.5.0", + "dom-serializer": "^1.3.2", + "domhandler": "^4.2.0", + "htmlparser2": "^6.1.0", + "parse5": "^6.0.1", + "parse5-htmlparser2-tree-adapter": "^6.0.1", + "tslib": "^2.2.0" + }, + "dependencies": { + "dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "dev": true, + "requires": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + } + }, + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true + }, + "parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "dev": true + } + } + }, + "cheerio-select": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/cheerio-select/-/cheerio-select-1.6.0.tgz", + "integrity": "sha512-eq0GdBvxVFbqWgmCm7M3XGs1I8oLy/nExUnh6oLqmBditPO9AqQJrkslDpMun/hZ0yyTs8L0m85OHp4ho6Qm9g==", + "dev": true, + "requires": { + "css-select": "^4.3.0", + "css-what": "^6.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.3.1", + "domutils": "^2.8.0" + }, + "dependencies": { + "css-select": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/css-select/-/css-select-4.3.0.tgz", + "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==", + "dev": true, + "requires": { + "boolbase": "^1.0.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", + "domutils": "^2.8.0", + "nth-check": "^2.0.1" + } + }, + "css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true + }, + "dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "dev": true, + "requires": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + } + }, + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true + }, + "domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmmirror.com/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "dev": true, + "requires": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + } + }, + "nth-check": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/nth-check/-/nth-check-2.0.1.tgz", + "integrity": "sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==", + "dev": true, + "requires": { + "boolbase": "^1.0.0" + } + } + } + }, + "chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "optional": true, + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "dependencies": { + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "optional": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "optional": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "optional": true + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "optional": true, + "requires": { + "is-number": "^7.0.0" + } + } + } + }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "chrome-trace-event": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", + "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==" + }, + "ci-info": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/ci-info/-/ci-info-1.6.0.tgz", + "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A==", + "dev": true + }, + "cipher-base": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/cipher-base/-/cipher-base-1.0.4.tgz", + "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "circular-json": { + "version": "0.3.3", + "resolved": "https://registry.npmmirror.com/circular-json/-/circular-json-0.3.3.tgz", + "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A==", + "dev": true, + "optional": true + }, + "class-utils": { + "version": "0.3.6", + "resolved": "https://registry.npmmirror.com/class-utils/-/class-utils-0.3.6.tgz", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "requires": { + "arr-union": "^3.1.0", + "define-property": "^0.2.5", + "isobject": "^3.0.0", + "static-extend": "^0.1.1" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "requires": { + "is-descriptor": "^0.1.0" + } + } + } + }, + "clean-css": { + "version": "4.2.4", + "resolved": "https://registry.npmmirror.com/clean-css/-/clean-css-4.2.4.tgz", + "integrity": "sha512-EJUDT7nDVFDvaQgAo2G/PJvxmp1o/c6iXLbswsBbUFXi1Nr+AjA2cKmfbKDMjMvzEe75g3P6JkaDDAKk96A85A==", + "dev": true, + "requires": { + "source-map": "~0.6.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "dev": true + }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", + "dev": true, + "requires": { + "restore-cursor": "^2.0.0" + } + }, + "cli-highlight": { + "version": "2.1.11", + "resolved": "https://registry.npmmirror.com/cli-highlight/-/cli-highlight-2.1.11.tgz", + "integrity": "sha512-9KDcoEVwyUXrjcJNvHD0NFc/hiwe/WPVYIleQh2O1N2Zro5gWJZ/K+3DGn8w8P/F6FxOgzyC5bxDyHIgCSPhGg==", + "dev": true, + "requires": { + "chalk": "^4.0.0", + "highlight.js": "^10.7.1", + "mz": "^2.4.0", + "parse5": "^5.1.1", + "parse5-htmlparser2-tree-adapter": "^6.0.0", + "yargs": "^16.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmmirror.com/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "cli-spinners": { + "version": "2.6.1", + "resolved": "https://registry.npmmirror.com/cli-spinners/-/cli-spinners-2.6.1.tgz", + "integrity": "sha512-x/5fWmGMnbKQAaNwN+UZlV79qBLM9JFnJuJ03gIi5whrob0xV0ofNVHy9DhwGdsMJQc2OKv0oGmLzvaqvAVv+g==", + "dev": true + }, + "cli-truncate": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/cli-truncate/-/cli-truncate-0.2.1.tgz", + "integrity": "sha512-f4r4yJnbT++qUPI9NR4XLDLq41gQ+uqnPItWG0F5ZkehuNiTTa3EY0S4AqTSUOeJ7/zU41oWPQSNkW5BqPL9bg==", + "dev": true, + "requires": { + "slice-ansi": "0.0.4", + "string-width": "^1.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==", + "dev": true, + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "slice-ansi": { + "version": "0.0.4", + "resolved": "https://registry.npmmirror.com/slice-ansi/-/slice-ansi-0.0.4.tgz", + "integrity": "sha512-up04hB2hR92PgjpyU3y/eg91yIBILyjVY26NvvciY3EVVPjybkMszMpXQ9QAkcS3I5rtJBDLoTxxg+qvW8c7rw==", + "dev": true + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==", + "dev": true, + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + } + } + }, + "cli-width": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", + "dev": true, + "optional": true + }, + "clipboardy": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/clipboardy/-/clipboardy-2.3.0.tgz", + "integrity": "sha512-mKhiIL2DrQIsuXMgBgnfEHOZOryC7kY7YO//TN6c63wlEm3NG5tz+YgY5rVi29KCmq/QQjKYvM7a19+MDOTHOQ==", + "dev": true, + "requires": { + "arch": "^2.1.1", + "execa": "^1.0.0", + "is-wsl": "^2.1.1" + }, + "dependencies": { + "is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, + "requires": { + "is-docker": "^2.0.0" + } + } + } + }, + "cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dev": true, + "requires": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + } + } + } + }, + "clone": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "dev": true + }, + "clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + } + }, + "cmd-shim": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/cmd-shim/-/cmd-shim-4.1.0.tgz", + "integrity": "sha512-lb9L7EM4I/ZRVuljLPEtUJOP+xiQVknZ4ZMpMgEp4JzNldPb27HU03hi6K1/6CoIuit/Zm/LQXySErFeXxDprw==", + "dev": true, + "requires": { + "mkdirp-infer-owner": "^2.0.0" + } + }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmmirror.com/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "optional": true + }, + "coa": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/coa/-/coa-2.0.2.tgz", + "integrity": "sha512-q5/jG+YQnSy4nRTV4F7lPepBJZ8qBNJJDBuJdoejDyLXgmL7IEo+Le2JDZudFTFt7mrCqIRaSjws4ygRCTCAXA==", + "requires": { + "@types/q": "^1.5.1", + "chalk": "^2.4.1", + "q": "^1.1.2" + } + }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==", + "dev": true + }, + "collection-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/collection-visit/-/collection-visit-1.0.0.tgz", + "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==", + "requires": { + "map-visit": "^1.0.0", + "object-visit": "^1.0.0" + } + }, + "color": { + "version": "3.2.1", + "resolved": "https://registry.npmmirror.com/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dev": true, + "requires": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dev": true, + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "columnify": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/columnify/-/columnify-1.6.0.tgz", + "integrity": "sha512-lomjuFZKfM6MSAnV9aCZC9sc0qGbmZdfygNv+nCpqVkSKdCxCklLtd16O0EILGkImHw9ZpHkAnHaB+8Zxq5W6Q==", + "dev": true, + "requires": { + "strip-ansi": "^6.0.1", + "wcwidth": "^1.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmmirror.com/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmmirror.com/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==" + }, + "compare-func": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/compare-func/-/compare-func-2.0.0.tgz", + "integrity": "sha512-zHig5N+tPWARooBnb0Zx1MFcdfpyJrfTJ3Y5L+IFvUm8rM74hHz66z0gw0x4tijh5CorKkKUCnW82R2vmpeCRA==", + "dev": true, + "requires": { + "array-ify": "^1.0.0", + "dot-prop": "^5.1.0" + } + }, + "component-emitter": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/component-emitter/-/component-emitter-1.3.0.tgz", + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" + }, + "compress-commons": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/compress-commons/-/compress-commons-2.1.1.tgz", + "integrity": "sha512-eVw6n7CnEMFzc3duyFVrQEuY1BlHR3rYsSztyG32ibGMW722i3C6IizEGMFmfMU+A+fALvBIwxN3czffTcdA+Q==", + "dev": true, + "requires": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^3.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^2.3.6" + } + }, + "compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmmirror.com/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dev": true, + "requires": { + "mime-db": ">= 1.43.0 < 2" + } + }, + "compression": { + "version": "1.7.4", + "resolved": "https://registry.npmmirror.com/compression/-/compression-1.7.4.tgz", + "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "dev": true, + "requires": { + "accepts": "~1.3.5", + "bytes": "3.0.0", + "compressible": "~2.0.16", + "debug": "2.6.9", + "on-headers": "~1.0.2", + "safe-buffer": "5.1.2", + "vary": "~1.1.2" + }, + "dependencies": { + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", + "dev": true + } + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmmirror.com/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmmirror.com/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", + "dev": true, + "requires": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, + "connect-history-api-fallback": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/connect-history-api-fallback/-/connect-history-api-fallback-1.6.0.tgz", + "integrity": "sha512-e54B99q/OUoH64zYYRf3HBP5z24G38h5D3qXu23JGRoigpX5Ss4r9ZnDk3g0Z8uQC2x2lPaJ+UlWBc1ZWBWdLg==", + "dev": true + }, + "console-browserify": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/console-browserify/-/console-browserify-1.2.0.tgz", + "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==" + }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "dev": true + }, + "consolidate": { + "version": "0.15.1", + "resolved": "https://registry.npmmirror.com/consolidate/-/consolidate-0.15.1.tgz", + "integrity": "sha512-DW46nrsMJgy9kqAbPt5rKaCr7uFtpo4mSUvLHIUbJEjm0vo+aY5QLwBUq3FK4tRnJr/X0Psc0C4jf/h+HtXSMw==", + "dev": true, + "requires": { + "bluebird": "^3.1.1" + } + }, + "constants-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/constants-browserify/-/constants-browserify-1.0.0.tgz", + "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==" + }, + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "requires": { + "safe-buffer": "5.2.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + } + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "dev": true + }, + "conventional-changelog-angular": { + "version": "5.0.13", + "resolved": "https://registry.npmmirror.com/conventional-changelog-angular/-/conventional-changelog-angular-5.0.13.tgz", + "integrity": "sha512-i/gipMxs7s8L/QeuavPF2hLnJgH6pEZAttySB6aiQLWcX3puWDL3ACVmvBhJGxnAy52Qc15ua26BufY6KpmrVA==", + "dev": true, + "requires": { + "compare-func": "^2.0.0", + "q": "^1.5.1" + } + }, + "conventional-changelog-core": { + "version": "4.2.4", + "resolved": "https://registry.npmmirror.com/conventional-changelog-core/-/conventional-changelog-core-4.2.4.tgz", + "integrity": "sha512-gDVS+zVJHE2v4SLc6B0sLsPiloR0ygU7HaDW14aNJE1v4SlqJPILPl/aJC7YdtRE4CybBf8gDwObBvKha8Xlyg==", + "dev": true, + "requires": { + "add-stream": "^1.0.0", + "conventional-changelog-writer": "^5.0.0", + "conventional-commits-parser": "^3.2.0", + "dateformat": "^3.0.0", + "get-pkg-repo": "^4.0.0", + "git-raw-commits": "^2.0.8", + "git-remote-origin-url": "^2.0.0", + "git-semver-tags": "^4.1.1", + "lodash": "^4.17.15", + "normalize-package-data": "^3.0.0", + "q": "^1.5.1", + "read-pkg": "^3.0.0", + "read-pkg-up": "^3.0.0", + "through2": "^4.0.0" + }, + "dependencies": { + "hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "normalize-package-data": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", + "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", + "dev": true, + "requires": { + "hosted-git-info": "^4.0.1", + "is-core-module": "^2.5.0", + "semver": "^7.3.4", + "validate-npm-package-license": "^3.0.1" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true + }, + "read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha512-BLq/cCO9two+lBgiTYNqD6GdtK8s4NpaWrl6/rCO9w0TUS8oJl7cmToOZfRYllKTISY6nt1U7jQ53brmKqY6BA==", + "dev": true, + "requires": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + }, + "dependencies": { + "hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", + "dev": true + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmmirror.com/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "through2": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dev": true, + "requires": { + "readable-stream": "3" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "conventional-changelog-preset-loader": { + "version": "2.3.4", + "resolved": "https://registry.npmmirror.com/conventional-changelog-preset-loader/-/conventional-changelog-preset-loader-2.3.4.tgz", + "integrity": "sha512-GEKRWkrSAZeTq5+YjUZOYxdHq+ci4dNwHvpaBC3+ENalzFWuCWa9EZXSuZBpkr72sMdKB+1fyDV4takK1Lf58g==", + "dev": true + }, + "conventional-changelog-writer": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/conventional-changelog-writer/-/conventional-changelog-writer-5.0.1.tgz", + "integrity": "sha512-5WsuKUfxW7suLblAbFnxAcrvf6r+0b7GvNaWUwUIk0bXMnENP/PEieGKVUQrjPqwPT4o3EPAASBXiY6iHooLOQ==", + "dev": true, + "requires": { + "conventional-commits-filter": "^2.0.7", + "dateformat": "^3.0.0", + "handlebars": "^4.7.7", + "json-stringify-safe": "^5.0.1", + "lodash": "^4.17.15", + "meow": "^8.0.0", + "semver": "^6.0.0", + "split": "^1.0.0", + "through2": "^4.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "through2": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dev": true, + "requires": { + "readable-stream": "3" + } + } + } + }, + "conventional-commits-filter": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/conventional-commits-filter/-/conventional-commits-filter-2.0.7.tgz", + "integrity": "sha512-ASS9SamOP4TbCClsRHxIHXRfcGCnIoQqkvAzCSbZzTFLfcTqJVugB0agRgsEELsqaeWgsXv513eS116wnlSSPA==", + "dev": true, + "requires": { + "lodash.ismatch": "^4.4.0", + "modify-values": "^1.0.0" + } + }, + "conventional-commits-parser": { + "version": "3.2.4", + "resolved": "https://registry.npmmirror.com/conventional-commits-parser/-/conventional-commits-parser-3.2.4.tgz", + "integrity": "sha512-nK7sAtfi+QXbxHCYfhpZsfRtaitZLIA6889kFIouLvz6repszQDgxBu7wf2WbU+Dco7sAnNCJYERCwt54WPC2Q==", + "dev": true, + "requires": { + "JSONStream": "^1.0.4", + "is-text-path": "^1.0.1", + "lodash": "^4.17.15", + "meow": "^8.0.0", + "split2": "^3.0.0", + "through2": "^4.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "through2": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dev": true, + "requires": { + "readable-stream": "3" + } + } + } + }, + "conventional-recommended-bump": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/conventional-recommended-bump/-/conventional-recommended-bump-6.1.0.tgz", + "integrity": "sha512-uiApbSiNGM/kkdL9GTOLAqC4hbptObFo4wW2QRyHsKciGAfQuLU1ShZ1BIVI/+K2BE/W1AWYQMCXAsv4dyKPaw==", + "dev": true, + "requires": { + "concat-stream": "^2.0.0", + "conventional-changelog-preset-loader": "^2.3.4", + "conventional-commits-filter": "^2.0.7", + "conventional-commits-parser": "^3.2.0", + "git-raw-commits": "^2.0.8", + "git-semver-tags": "^4.1.1", + "meow": "^8.0.0", + "q": "^1.5.1" + }, + "dependencies": { + "concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "convert-source-map": { + "version": "1.8.0", + "resolved": "https://registry.npmmirror.com/convert-source-map/-/convert-source-map-1.8.0.tgz", + "integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.1" + } + }, + "cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmmirror.com/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "dev": true + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true + }, + "copy-anything": { + "version": "2.0.6", + "resolved": "https://registry.npmmirror.com/copy-anything/-/copy-anything-2.0.6.tgz", + "integrity": "sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==", + "dev": true, + "requires": { + "is-what": "^3.14.1" + } + }, + "copy-concurrently": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/copy-concurrently/-/copy-concurrently-1.0.5.tgz", + "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "requires": { + "aproba": "^1.1.1", + "fs-write-stream-atomic": "^1.0.8", + "iferr": "^0.1.5", + "mkdirp": "^0.5.1", + "rimraf": "^2.5.4", + "run-queue": "^1.0.0" + } + }, + "copy-descriptor": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz", + "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==" + }, + "copy-webpack-plugin": { + "version": "4.6.0", + "resolved": "https://registry.npmmirror.com/copy-webpack-plugin/-/copy-webpack-plugin-4.6.0.tgz", + "integrity": "sha512-Y+SQCF+0NoWQryez2zXn5J5knmr9z/9qSQt7fbL78u83rxmigOy8X5+BFn8CFSuX+nKT8gpYwJX68ekqtQt6ZA==", + "dev": true, + "requires": { + "cacache": "^10.0.4", + "find-cache-dir": "^1.0.0", + "globby": "^7.1.1", + "is-glob": "^4.0.0", + "loader-utils": "^1.1.0", + "minimatch": "^3.0.4", + "p-limit": "^1.0.0", + "serialize-javascript": "^1.4.0" + }, + "dependencies": { + "cacache": { + "version": "10.0.4", + "resolved": "https://registry.npmmirror.com/cacache/-/cacache-10.0.4.tgz", + "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==", + "dev": true, + "requires": { + "bluebird": "^3.5.1", + "chownr": "^1.0.1", + "glob": "^7.1.2", + "graceful-fs": "^4.1.11", + "lru-cache": "^4.1.1", + "mississippi": "^2.0.0", + "mkdirp": "^0.5.1", + "move-concurrently": "^1.0.1", + "promise-inflight": "^1.0.1", + "rimraf": "^2.6.2", + "ssri": "^5.2.4", + "unique-filename": "^1.1.0", + "y18n": "^4.0.0" + } + }, + "find-cache-dir": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/find-cache-dir/-/find-cache-dir-1.0.0.tgz", + "integrity": "sha512-46TFiBOzX7xq/PcSWfFwkyjpemdRnMe31UQF+os0y+1W3k95f6R4SEt02Hj4p3X0Mir9gfrkmOtshFidS0VPUg==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^1.0.0", + "pkg-dir": "^2.0.0" + } + }, + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, + "globby": { + "version": "7.1.1", + "resolved": "https://registry.npmmirror.com/globby/-/globby-7.1.1.tgz", + "integrity": "sha512-yANWAN2DUcBtuus5Cpd+SKROzXHs2iVXFZt/Ykrfz6SAXqacLX25NZpltE+39ceMexYF4TtEadjuSTw8+3wX4g==", + "dev": true, + "requires": { + "array-union": "^1.0.1", + "dir-glob": "^2.0.0", + "glob": "^7.1.2", + "ignore": "^3.3.5", + "pify": "^3.0.0", + "slash": "^1.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "mississippi": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/mississippi/-/mississippi-2.0.0.tgz", + "integrity": "sha512-zHo8v+otD1J10j/tC+VNoGK9keCuByhKovAvdn74dmxJl9+mWHnx6EMsDN4lgRoMI/eYo2nchAxniIbUPb5onw==", + "dev": true, + "requires": { + "concat-stream": "^1.5.0", + "duplexify": "^3.4.2", + "end-of-stream": "^1.1.0", + "flush-write-stream": "^1.0.0", + "from2": "^2.1.0", + "parallel-transform": "^1.1.0", + "pump": "^2.0.1", + "pumpify": "^1.3.3", + "stream-each": "^1.1.0", + "through2": "^2.0.0" + } + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", + "dev": true + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true + }, + "pkg-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/pkg-dir/-/pkg-dir-2.0.0.tgz", + "integrity": "sha512-ojakdnUgL5pzJYWw2AIDEupaQCX5OPbM688ZevubICjdIX01PRSYKqm33fJoCOJBRseYCTUlQRnBNX+Pchaejw==", + "dev": true, + "requires": { + "find-up": "^2.1.0" + } + }, + "pump": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "serialize-javascript": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/serialize-javascript/-/serialize-javascript-1.9.1.tgz", + "integrity": "sha512-0Vb/54WJ6k5v8sSWN09S0ora+Hnr+cX40r9F170nT+mSkaxltoE/7R3OrIdBSUv1OoiobH1QoWQbCnAO+e8J1A==", + "dev": true + }, + "slash": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/slash/-/slash-1.0.0.tgz", + "integrity": "sha512-3TYDR7xWt4dIqV2JauJr+EJeW356RXijHeUlO+8djJ+uBXPn8/2dpzBc8yQhh583sVvc9CvFAeQVgijsH+PNNg==", + "dev": true + }, + "ssri": { + "version": "5.3.0", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-5.3.0.tgz", + "integrity": "sha512-XRSIPqLij52MtgoQavH/x/dU1qVKtWUAAZeOHsR9c2Ddi4XerFy3mc1alf+dLJKl9EUIm/Ht+EowFkTUOA6GAQ==", + "dev": true, + "requires": { + "safe-buffer": "^5.1.1" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", + "dev": true + } + } + }, + "core-js": { + "version": "2.6.11", + "resolved": "https://registry.npmmirror.com/core-js/-/core-js-2.6.11.tgz", + "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==" + }, + "core-js-compat": { + "version": "3.22.2", + "resolved": "https://registry.npmmirror.com/core-js-compat/-/core-js-compat-3.22.2.tgz", + "integrity": "sha512-Fns9lU06ZJ07pdfmPMu7OnkIKGPKDzXKIiuGlSvHHapwqMUF2QnnsWwtueFZtSyZEilP0o6iUeHQwpn7LxtLUw==", + "dev": true, + "requires": { + "browserslist": "^4.20.2", + "semver": "7.0.0" + }, + "dependencies": { + "semver": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "dev": true + } + } + }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "cosmiconfig": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz", + "integrity": "sha512-H65gsXo1SKjf8zmrJ67eJk8aIRKV5ff2D4uKZIBZShbhGSpEmsQOPW/SKMKYhSTrqR7ufy6RP69rPogdaPh/kA==", + "dev": true, + "requires": { + "import-fresh": "^2.0.0", + "is-directory": "^0.3.1", + "js-yaml": "^3.13.1", + "parse-json": "^4.0.0" + } + }, + "cpx": { + "version": "1.5.0", + "resolved": "https://registry.npmmirror.com/cpx/-/cpx-1.5.0.tgz", + "integrity": "sha512-jHTjZhsbg9xWgsP2vuNW2jnnzBX+p4T+vNI9Lbjzs1n4KhOfa22bQppiFYLsWQKd8TzmL5aSP/Me3yfsCwXbDA==", + "dev": true, + "requires": { + "babel-runtime": "^6.9.2", + "chokidar": "^1.6.0", + "duplexer": "^0.1.1", + "glob": "^7.0.5", + "glob2base": "^0.0.12", + "minimatch": "^3.0.2", + "mkdirp": "^0.5.1", + "resolve": "^1.1.7", + "safe-buffer": "^5.0.1", + "shell-quote": "^1.6.1", + "subarg": "^1.0.0" + }, + "dependencies": { + "anymatch": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-1.3.2.tgz", + "integrity": "sha512-0XNayC8lTHQ2OI8aljNCN3sSx6hsr/1+rlcDAotXJR7C1oZZHCNsfpbKwMjRA3Uqb5tF1Rae2oloTr4xpq+WjA==", + "dev": true, + "requires": { + "micromatch": "^2.1.5", + "normalize-path": "^2.0.0" + } + }, + "arr-diff": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/arr-diff/-/arr-diff-2.0.0.tgz", + "integrity": "sha512-dtXTVMkh6VkEEA7OhXnN1Ecb8aAGFdZ1LFxtOCoqj4qkyOJMt7+qs6Ahdy6p/NQCPYsRSXXivhSB/J5E9jmYKA==", + "dev": true, + "requires": { + "arr-flatten": "^1.0.1" + } + }, + "array-unique": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/array-unique/-/array-unique-0.2.1.tgz", + "integrity": "sha512-G2n5bG5fSUCpnsXz4+8FUkYsGPkNfLn9YvS66U5qbTIXI2Ynnlo4Bi42bWv+omKUCqz+ejzfClwne0alJWJPhg==", + "dev": true + }, + "binary-extensions": { + "version": "1.13.1", + "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-1.13.1.tgz", + "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", + "dev": true + }, + "braces": { + "version": "1.8.5", + "resolved": "https://registry.npmmirror.com/braces/-/braces-1.8.5.tgz", + "integrity": "sha512-xU7bpz2ytJl1bH9cgIurjpg/n8Gohy9GTw81heDYLJQ4RU60dlyJsa+atVF2pI0yMMvKxI9HkKwjePCj5XI1hw==", + "dev": true, + "requires": { + "expand-range": "^1.8.1", + "preserve": "^0.2.0", + "repeat-element": "^1.1.2" + } + }, + "chokidar": { + "version": "1.7.0", + "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-1.7.0.tgz", + "integrity": "sha512-mk8fAWcRUOxY7btlLtitj3A45jOwSAxH4tOFOoEGbVsl6cL6pPMWUy7dwZ/canfj3QEdP6FHSnf/l1c6/WkzVg==", + "dev": true, + "requires": { + "anymatch": "^1.3.0", + "async-each": "^1.0.0", + "fsevents": "^1.0.0", + "glob-parent": "^2.0.0", + "inherits": "^2.0.1", + "is-binary-path": "^1.0.0", + "is-glob": "^2.0.0", + "path-is-absolute": "^1.0.0", + "readdirp": "^2.0.0" + } + }, + "expand-brackets": { + "version": "0.1.5", + "resolved": "https://registry.npmmirror.com/expand-brackets/-/expand-brackets-0.1.5.tgz", + "integrity": "sha512-hxx03P2dJxss6ceIeri9cmYOT4SRs3Zk3afZwWpOsRqLqprhTR8u++SlC+sFGsQr7WGFPdMF7Gjc1njDLDK6UA==", + "dev": true, + "requires": { + "is-posix-bracket": "^0.1.0" + } + }, + "extglob": { + "version": "0.3.2", + "resolved": "https://registry.npmmirror.com/extglob/-/extglob-0.3.2.tgz", + "integrity": "sha512-1FOj1LOwn42TMrruOHGt18HemVnbwAmAak7krWk+wa93KXxGbK+2jpezm+ytJYDaBX0/SPLZFHKM7m+tKobWGg==", + "dev": true, + "requires": { + "is-extglob": "^1.0.0" + } + }, + "fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "dev": true, + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } + }, + "glob-parent": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-2.0.0.tgz", + "integrity": "sha512-JDYOvfxio/t42HKdxkAYaCiBN7oYiuxykOxKxdaUW5Qn0zaYN3gRQWolrwdnf0shM9/EP0ebuuTmyoXNr1cC5w==", + "dev": true, + "requires": { + "is-glob": "^2.0.0" + } + }, + "is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "requires": { + "kind-of": "^6.0.0" + }, + "dependencies": { + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + } + } + }, + "is-binary-path": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-1.0.1.tgz", + "integrity": "sha512-9fRVlXc0uCxEDj1nQzaWONSpbTfx0FmJfzHF7pwlI8DkWGoHBBea4Pg5Ky0ojwwxQmnSifgbKkI06Qv0Ljgj+Q==", + "dev": true, + "requires": { + "binary-extensions": "^1.0.0" + } + }, + "is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "requires": { + "kind-of": "^6.0.0" + }, + "dependencies": { + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + } + } + }, + "is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "dependencies": { + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + } + } + }, + "is-extglob": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-extglob/-/is-extglob-1.0.0.tgz", + "integrity": "sha512-7Q+VbVafe6x2T+Tu6NcOf6sRklazEPmBoB3IWk3WdGZM2iGUwU/Oe3Wtq5lSEkDTTlpp8yx+5t4pzO/i9Ty1ww==", + "dev": true + }, + "is-glob": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-2.0.1.tgz", + "integrity": "sha512-a1dBeB19NXsf/E0+FHqkagizel/LQw2DjSQpvQrj3zT+jYPpaUCryPnrQajXKFLCMuf4I6FhRpaGtw4lPrG6Eg==", + "dev": true, + "requires": { + "is-extglob": "^1.0.0" + } + }, + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + }, + "micromatch": { + "version": "2.3.11", + "resolved": "https://registry.npmmirror.com/micromatch/-/micromatch-2.3.11.tgz", + "integrity": "sha512-LnU2XFEk9xxSJ6rfgAry/ty5qwUTyHYOBU0g4R6tIw5ljwgGIBmiKhRWLw5NpMOnrgUNcDJ4WMp8rl3sYVHLNA==", + "dev": true, + "requires": { + "arr-diff": "^2.0.0", + "array-unique": "^0.2.1", + "braces": "^1.8.2", + "expand-brackets": "^0.1.4", + "extglob": "^0.3.1", + "filename-regex": "^2.0.0", + "is-extglob": "^1.0.0", + "is-glob": "^2.0.1", + "kind-of": "^3.0.2", + "normalize-path": "^2.0.1", + "object.omit": "^2.0.0", + "parse-glob": "^3.0.4", + "regex-cache": "^0.4.2" + } + }, + "normalize-path": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-2.1.1.tgz", + "integrity": "sha512-3pKJwH184Xo/lnH6oyP1q2pMd7HcypqqmRs91/6/i2CGtWwIKGCkOOMTm/zXbgTEWHw1uNpNi/igc3ePOYHb6w==", + "dev": true, + "requires": { + "remove-trailing-separator": "^1.0.1" + } + }, + "readdirp": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-2.2.1.tgz", + "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11", + "micromatch": "^3.1.10", + "readable-stream": "^2.0.2" + }, + "dependencies": { + "arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==", + "dev": true + }, + "array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmmirror.com/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==", + "dev": true + }, + "braces": { + "version": "2.3.2", + "resolved": "https://registry.npmmirror.com/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "requires": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmmirror.com/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", + "dev": true, + "requires": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmmirror.com/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + } + } + }, + "extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, + "requires": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmmirror.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + }, + "micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmmirror.com/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + } + } + } + } + } + }, + "crc": { + "version": "3.8.0", + "resolved": "https://registry.npmmirror.com/crc/-/crc-3.8.0.tgz", + "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==", + "dev": true, + "requires": { + "buffer": "^5.1.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + } + } + }, + "crc32-stream": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/crc32-stream/-/crc32-stream-3.0.1.tgz", + "integrity": "sha512-mctvpXlbzsvK+6z8kJwSJ5crm7yBwrQMTybJzMw1O4lLGJqjlDCXY2Zw7KheiA6XBEcBmfLx1D88mjRGVJtY9w==", + "dev": true, + "requires": { + "crc": "^3.4.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "create-ecdh": { + "version": "4.0.4", + "resolved": "https://registry.npmmirror.com/create-ecdh/-/create-ecdh-4.0.4.tgz", + "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", + "requires": { + "bn.js": "^4.1.0", + "elliptic": "^6.5.3" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmmirror.com/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "create-hash": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/create-hash/-/create-hash-1.2.0.tgz", + "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "requires": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "md5.js": "^1.3.4", + "ripemd160": "^2.0.1", + "sha.js": "^2.4.0" + } + }, + "create-hmac": { + "version": "1.1.7", + "resolved": "https://registry.npmmirror.com/create-hmac/-/create-hmac-1.1.7.tgz", + "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "requires": { + "cipher-base": "^1.0.3", + "create-hash": "^1.1.0", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmmirror.com/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==" + }, + "crypto-browserify": { + "version": "3.12.0", + "resolved": "https://registry.npmmirror.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz", + "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "requires": { + "browserify-cipher": "^1.0.0", + "browserify-sign": "^4.0.0", + "create-ecdh": "^4.0.0", + "create-hash": "^1.1.0", + "create-hmac": "^1.1.0", + "diffie-hellman": "^5.0.0", + "inherits": "^2.0.1", + "pbkdf2": "^3.0.3", + "public-encrypt": "^4.0.0", + "randombytes": "^2.0.0", + "randomfill": "^1.0.3" + } + }, + "csp-html-webpack-plugin": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/csp-html-webpack-plugin/-/csp-html-webpack-plugin-4.0.0.tgz", + "integrity": "sha512-1YqQefNG0SrZisysThlly2bgs4Ab/W91xOM17S8wd+6vTo3E0OdL+y4IAR0MKpthRluNGzFB3QhPqdOhkXAExg==", + "dev": true, + "requires": { + "cheerio": "^1.0.0-rc.3", + "lodash": "^4.17.15", + "memory-fs": "^0.5.0" + }, + "dependencies": { + "memory-fs": { + "version": "0.5.0", + "resolved": "https://registry.npmmirror.com/memory-fs/-/memory-fs-0.5.0.tgz", + "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==", + "dev": true, + "requires": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + } + } + } + }, + "css-color-names": { + "version": "0.0.4", + "resolved": "https://registry.npmmirror.com/css-color-names/-/css-color-names-0.0.4.tgz", + "integrity": "sha512-zj5D7X1U2h2zsXOAM8EyUREBnnts6H+Jm+d1M2DbiQQcUtnqgQsMrdo8JW9R80YFUmIdBZeMu5wvYM7hcgWP/Q==", + "dev": true + }, + "css-declaration-sorter": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/css-declaration-sorter/-/css-declaration-sorter-4.0.1.tgz", + "integrity": "sha512-BcxQSKTSEEQUftYpBVnsH4SF05NTuBokb19/sBt6asXGKZ/6VP7PLG1CBCkFDYOnhXhPh0jMhO6xZ71oYHXHBA==", + "dev": true, + "requires": { + "postcss": "^7.0.1", + "timsort": "^0.3.0" + } + }, + "css-loader": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/css-loader/-/css-loader-1.0.1.tgz", + "integrity": "sha512-+ZHAZm/yqvJ2kDtPne3uX0C+Vr3Zn5jFn2N4HywtS5ujwvsVkyg0VArEXpl3BgczDA8anieki1FIzhchX4yrDw==", + "dev": true, + "requires": { + "babel-code-frame": "^6.26.0", + "css-selector-tokenizer": "^0.7.0", + "icss-utils": "^2.1.0", + "loader-utils": "^1.0.2", + "lodash": "^4.17.11", + "postcss": "^6.0.23", + "postcss-modules-extract-imports": "^1.2.0", + "postcss-modules-local-by-default": "^1.2.0", + "postcss-modules-scope": "^1.1.0", + "postcss-modules-values": "^1.3.0", + "postcss-value-parser": "^3.3.0", + "source-list-map": "^2.0.0" + }, + "dependencies": { + "postcss": { + "version": "6.0.23", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-6.0.23.tgz", + "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==", + "dev": true, + "requires": { + "chalk": "^2.4.1", + "source-map": "^0.6.1", + "supports-color": "^5.4.0" + } + }, + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "css-select": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/css-select/-/css-select-2.1.0.tgz", + "integrity": "sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ==", + "requires": { + "boolbase": "^1.0.0", + "css-what": "^3.2.1", + "domutils": "^1.7.0", + "nth-check": "^1.0.2" + } + }, + "css-select-base-adapter": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/css-select-base-adapter/-/css-select-base-adapter-0.1.1.tgz", + "integrity": "sha512-jQVeeRG70QI08vSTwf1jHxp74JoZsr2XSgETae8/xC8ovSnL2WF87GTLO86Sbwdt2lK4Umg4HnnwMO4YF3Ce7w==" + }, + "css-selector-tokenizer": { + "version": "0.7.3", + "resolved": "https://registry.npmmirror.com/css-selector-tokenizer/-/css-selector-tokenizer-0.7.3.tgz", + "integrity": "sha512-jWQv3oCEL5kMErj4wRnK/OPoBi0D+P1FR2cDCKYPaMeD2eW3/mttav8HT4hT1CKopiJI/psEULjkClhvJo4Lvg==", + "dev": true, + "requires": { + "cssesc": "^3.0.0", + "fastparse": "^1.1.2" + } + }, + "css-tree": { + "version": "1.0.0-alpha.33", + "resolved": "https://registry.npmmirror.com/css-tree/-/css-tree-1.0.0-alpha.33.tgz", + "integrity": "sha512-SPt57bh5nQnpsTBsx/IXbO14sRc9xXu5MtMAVuo0BaQQmyf0NupNPPSoMaqiAF5tDFafYsTkfeH4Q/HCKXkg4w==", + "requires": { + "mdn-data": "2.0.4", + "source-map": "^0.5.3" + } + }, + "css-what": { + "version": "3.4.2", + "resolved": "https://registry.npmmirror.com/css-what/-/css-what-3.4.2.tgz", + "integrity": "sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==" + }, + "cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true + }, + "cssnano": { + "version": "4.1.11", + "resolved": "https://registry.npmmirror.com/cssnano/-/cssnano-4.1.11.tgz", + "integrity": "sha512-6gZm2htn7xIPJOHY824ERgj8cNPgPxyCSnkXc4v7YvNW+TdVfzgngHcEhy/8D11kUWRUMbke+tC+AUcUsnMz2g==", + "dev": true, + "requires": { + "cosmiconfig": "^5.0.0", + "cssnano-preset-default": "^4.0.8", + "is-resolvable": "^1.0.0", + "postcss": "^7.0.0" + } + }, + "cssnano-preset-default": { + "version": "4.0.8", + "resolved": "https://registry.npmmirror.com/cssnano-preset-default/-/cssnano-preset-default-4.0.8.tgz", + "integrity": "sha512-LdAyHuq+VRyeVREFmuxUZR1TXjQm8QQU/ktoo/x7bz+SdOge1YKc5eMN6pRW7YWBmyq59CqYba1dJ5cUukEjLQ==", + "dev": true, + "requires": { + "css-declaration-sorter": "^4.0.1", + "cssnano-util-raw-cache": "^4.0.1", + "postcss": "^7.0.0", + "postcss-calc": "^7.0.1", + "postcss-colormin": "^4.0.3", + "postcss-convert-values": "^4.0.1", + "postcss-discard-comments": "^4.0.2", + "postcss-discard-duplicates": "^4.0.2", + "postcss-discard-empty": "^4.0.1", + "postcss-discard-overridden": "^4.0.1", + "postcss-merge-longhand": "^4.0.11", + "postcss-merge-rules": "^4.0.3", + "postcss-minify-font-values": "^4.0.2", + "postcss-minify-gradients": "^4.0.2", + "postcss-minify-params": "^4.0.2", + "postcss-minify-selectors": "^4.0.2", + "postcss-normalize-charset": "^4.0.1", + "postcss-normalize-display-values": "^4.0.2", + "postcss-normalize-positions": "^4.0.2", + "postcss-normalize-repeat-style": "^4.0.2", + "postcss-normalize-string": "^4.0.2", + "postcss-normalize-timing-functions": "^4.0.2", + "postcss-normalize-unicode": "^4.0.1", + "postcss-normalize-url": "^4.0.1", + "postcss-normalize-whitespace": "^4.0.2", + "postcss-ordered-values": "^4.1.2", + "postcss-reduce-initial": "^4.0.3", + "postcss-reduce-transforms": "^4.0.2", + "postcss-svgo": "^4.0.3", + "postcss-unique-selectors": "^4.0.1" + } + }, + "cssnano-util-get-arguments": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/cssnano-util-get-arguments/-/cssnano-util-get-arguments-4.0.0.tgz", + "integrity": "sha512-6RIcwmV3/cBMG8Aj5gucQRsJb4vv4I4rn6YjPbVWd5+Pn/fuG+YseGvXGk00XLkoZkaj31QOD7vMUpNPC4FIuw==", + "dev": true + }, + "cssnano-util-get-match": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/cssnano-util-get-match/-/cssnano-util-get-match-4.0.0.tgz", + "integrity": "sha512-JPMZ1TSMRUPVIqEalIBNoBtAYbi8okvcFns4O0YIhcdGebeYZK7dMyHJiQ6GqNBA9kE0Hym4Aqym5rPdsV/4Cw==", + "dev": true + }, + "cssnano-util-raw-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/cssnano-util-raw-cache/-/cssnano-util-raw-cache-4.0.1.tgz", + "integrity": "sha512-qLuYtWK2b2Dy55I8ZX3ky1Z16WYsx544Q0UWViebptpwn/xDBmog2TLg4f+DBMg1rJ6JDWtn96WHbOKDWt1WQA==", + "dev": true, + "requires": { + "postcss": "^7.0.0" + } + }, + "cssnano-util-same-parent": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/cssnano-util-same-parent/-/cssnano-util-same-parent-4.0.1.tgz", + "integrity": "sha512-WcKx5OY+KoSIAxBW6UBBRay1U6vkYheCdjyVNDm85zt5K9mHoGOfsOsqIszfAqrQQFIIKgjh2+FDgIj/zsl21Q==", + "dev": true + }, + "csso": { + "version": "3.5.1", + "resolved": "https://registry.npmmirror.com/csso/-/csso-3.5.1.tgz", + "integrity": "sha512-vrqULLffYU1Q2tLdJvaCYbONStnfkfimRxXNaGjxMldI0C7JPBC4rB1RyjhfdZ4m1frm8pM9uRPKH3d2knZ8gg==", + "requires": { + "css-tree": "1.0.0-alpha.29" + }, + "dependencies": { + "css-tree": { + "version": "1.0.0-alpha.29", + "resolved": "https://registry.npmmirror.com/css-tree/-/css-tree-1.0.0-alpha.29.tgz", + "integrity": "sha512-sRNb1XydwkW9IOci6iB2xmy8IGCj6r/fr+JWitvJ2JxQRPzN3T4AGGVWCMlVmVwM1gtgALJRmGIlWv5ppnGGkg==", + "requires": { + "mdn-data": "~1.1.0", + "source-map": "^0.5.3" + } + }, + "mdn-data": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/mdn-data/-/mdn-data-1.1.4.tgz", + "integrity": "sha512-FSYbp3lyKjyj3E7fMl6rYvUdX0FBXaluGqlFoYESWQlyUTq8R+wp0rkFxoYFqZlHCvsUXGjyJmLQSnXToYhOSA==" + } + } + }, + "csstype": { + "version": "3.0.11", + "resolved": "https://registry.npmmirror.com/csstype/-/csstype-3.0.11.tgz", + "integrity": "sha512-sa6P2wJ+CAbgyy4KFssIb/JNMLxFvKF1pCYCSXS8ZMuqZnMsrxqI2E5sPyoTpxoPU/gVZMzr2zjOfg8GIZOMsw==" + }, + "current-script-polyfill": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/current-script-polyfill/-/current-script-polyfill-1.0.0.tgz", + "integrity": "sha512-qv8s+G47V6Hq+g2kRE5th+ASzzrL7b6l+tap1DHKK25ZQJv3yIFhH96XaQ7NGL+zRW3t/RDbweJf/dJDe5Z5KA==", + "dev": true + }, + "currently-unhandled": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz", + "integrity": "sha512-/fITjgjGU50vjQ4FH6eUoYu+iUoUKIXws2hL15JJpIR+BbTxaXQsMuuyjtNh2WqsSBS5nsaZHFsFecyw5CCAng==", + "dev": true, + "requires": { + "array-find-index": "^1.0.1" + } + }, + "cyclist": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/cyclist/-/cyclist-1.0.1.tgz", + "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==" + }, + "d3-dispatch": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/d3-dispatch/-/d3-dispatch-2.0.0.tgz", + "integrity": "sha512-S/m2VsXI7gAti2pBoLClFFTMOO1HTtT0j99AuXLoGFKO6deHDdnv6ZGTxSTTUTgO1zVcv82fCOtDjYK4EECmWA==" + }, + "d3-force": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/d3-force/-/d3-force-2.1.1.tgz", + "integrity": "sha512-nAuHEzBqMvpFVMf9OX75d00OxvOXdxY+xECIXjW6Gv8BRrXu6gAWbv/9XKrvfJ5i5DCokDW7RYE50LRoK092ew==", + "requires": { + "d3-dispatch": "1 - 2", + "d3-quadtree": "1 - 2", + "d3-timer": "1 - 2" + } + }, + "d3-quadtree": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/d3-quadtree/-/d3-quadtree-2.0.0.tgz", + "integrity": "sha512-b0Ed2t1UUalJpc3qXzKi+cPGxeXRr4KU9YSlocN74aTzp6R/Ud43t79yLLqxHRWZfsvWXmbDWPpoENK1K539xw==" + }, + "d3-timer": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/d3-timer/-/d3-timer-2.0.0.tgz", + "integrity": "sha512-TO4VLh0/420Y/9dO3+f9abDEFYeCUr2WZRlxJvbp4HPTQcSylXNiL6yZa9FIUvV1yRiFufl1bszTCLDqv9PWNA==" + }, + "dagre": { + "version": "0.8.5", + "resolved": "https://registry.npmmirror.com/dagre/-/dagre-0.8.5.tgz", + "integrity": "sha512-/aTqmnRta7x7MCCpExk7HQL2O4owCT2h8NT//9I1OQ9vt29Pa0BzSAkR5lwFUcQ7491yVi/3CXU9jQ5o0Mn2Sw==", + "requires": { + "graphlib": "^2.1.8", + "lodash": "^4.17.15" + } + }, + "dargs": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/dargs/-/dargs-7.0.0.tgz", + "integrity": "sha512-2iy1EkLdlBzQGvbweYRFxmFath8+K7+AKB0TlhHWkNuH+TmovaMH/Wp7V7R4u7f4SnX3OgLsU9t1NI9ioDnUpg==", + "dev": true + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "date-fns": { + "version": "1.30.1", + "resolved": "https://registry.npmmirror.com/date-fns/-/date-fns-1.30.1.tgz", + "integrity": "sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==", + "dev": true + }, + "dateformat": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/dateformat/-/dateformat-3.0.3.tgz", + "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==", + "dev": true + }, + "de-indent": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/de-indent/-/de-indent-1.0.2.tgz", + "integrity": "sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==", + "dev": true + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmmirror.com/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "debuglog": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/debuglog/-/debuglog-1.0.1.tgz", + "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==", + "dev": true + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true + }, + "decamelize-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/decamelize-keys/-/decamelize-keys-1.1.0.tgz", + "integrity": "sha512-ocLWuYzRPoS9bfiSdDd3cxvrzovVMZnRDVEzAs+hWIVXGDbHxWMECij2OBuyB/An0FFW/nLuq6Kv1i/YC5Qfzg==", + "dev": true, + "requires": { + "decamelize": "^1.1.0", + "map-obj": "^1.0.0" + }, + "dependencies": { + "map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", + "dev": true + } + } + }, + "decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==" + }, + "dedent": { + "version": "0.7.0", + "resolved": "https://registry.npmmirror.com/dedent/-/dedent-0.7.0.tgz", + "integrity": "sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==", + "dev": true + }, + "deep-equal": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/deep-equal/-/deep-equal-1.1.1.tgz", + "integrity": "sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g==", + "dev": true, + "requires": { + "is-arguments": "^1.0.4", + "is-date-object": "^1.0.1", + "is-regex": "^1.0.4", + "object-is": "^1.0.1", + "object-keys": "^1.1.1", + "regexp.prototype.flags": "^1.2.0" + } + }, + "deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true + }, + "deepmerge": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/deepmerge/-/deepmerge-2.2.1.tgz", + "integrity": "sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA==" + }, + "default-gateway": { + "version": "5.0.5", + "resolved": "https://registry.npmmirror.com/default-gateway/-/default-gateway-5.0.5.tgz", + "integrity": "sha512-z2RnruVmj8hVMmAnEJMTIJNijhKCDiGjbLP+BHJFOT7ld3Bo5qcIBpVYDniqhbMIIf+jZDlkP2MkPXiQy/DBLA==", + "dev": true, + "requires": { + "execa": "^3.3.0" + }, + "dependencies": { + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "execa": { + "version": "3.4.0", + "resolved": "https://registry.npmmirror.com/execa/-/execa-3.4.0.tgz", + "integrity": "sha512-r9vdGQk4bmCuK1yKQu1KTwcT2zwfWdbdaXfCtAh+5nU/4fSX+JAb7vZGvI5naJrQlvONrEB20jeruESI69530g==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.0", + "get-stream": "^5.0.0", + "human-signals": "^1.1.1", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.0", + "onetime": "^5.1.0", + "p-finally": "^2.0.0", + "signal-exit": "^3.0.2", + "strip-final-newline": "^2.0.0" + } + }, + "get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "requires": { + "path-key": "^3.0.0" + } + }, + "onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "p-finally": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/p-finally/-/p-finally-2.0.1.tgz", + "integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw==", + "dev": true + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, + "defaults": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/defaults/-/defaults-1.0.3.tgz", + "integrity": "sha512-s82itHOnYrN0Ib8r+z7laQz3sdE+4FP3d9Q7VLO7U+KRT+CR0GsWuyHxzdAY82I7cXv0G/twrqomTJLOssO5HA==", + "dev": true, + "requires": { + "clone": "^1.0.2" + } + }, + "define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "requires": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + } + }, + "define-property": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-2.0.2.tgz", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "requires": { + "is-descriptor": "^1.0.2", + "isobject": "^3.0.1" + }, + "dependencies": { + "is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "requires": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + } + } + }, + "del": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/del/-/del-4.1.1.tgz", + "integrity": "sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==", + "dev": true, + "requires": { + "@types/glob": "^7.1.1", + "globby": "^6.1.0", + "is-path-cwd": "^2.0.0", + "is-path-in-cwd": "^2.0.0", + "p-map": "^2.0.0", + "pify": "^4.0.1", + "rimraf": "^2.6.3" + }, + "dependencies": { + "globby": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/globby/-/globby-6.1.0.tgz", + "integrity": "sha512-KVbFv2TQtbzCoxAnfD6JcHZTYCzyliEaaeM/gH8qQdkKr5s0OP9scEgvdcngyk7AVdY6YVW/TJHd+lQ/Df3Daw==", + "dev": true, + "requires": { + "array-union": "^1.0.1", + "glob": "^7.0.3", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true + } + } + } + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true + }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "dev": true + }, + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true + }, + "deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmmirror.com/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==", + "dev": true + }, + "des.js": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/des.js/-/des.js-1.0.1.tgz", + "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==", + "requires": { + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, + "destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true + }, + "detect-indent": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/detect-indent/-/detect-indent-5.0.0.tgz", + "integrity": "sha512-rlpvsxUtM0PQvy9iZe640/IWwWYyBsTApREbA1pHOpmOUIl9MkP/U4z7vTtg4Oaojvqhxt7sdufnT0EzGaR31g==", + "dev": true + }, + "detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true + }, + "dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "dev": true, + "requires": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, + "diffie-hellman": { + "version": "5.0.3", + "resolved": "https://registry.npmmirror.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", + "requires": { + "bn.js": "^4.1.0", + "miller-rabin": "^4.0.0", + "randombytes": "^2.0.0" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmmirror.com/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "dir-glob": { + "version": "2.2.2", + "resolved": "https://registry.npmmirror.com/dir-glob/-/dir-glob-2.2.2.tgz", + "integrity": "sha512-f9LBi5QWzIW3I6e//uxZoLBlUt9kcp66qo0sSCxL6YZKc75R1c4MFCoe/LaZiBGmgujvQdxc5Bn3QhfyvK5Hsw==", + "dev": true, + "requires": { + "path-type": "^3.0.0" + } + }, + "dns-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/dns-equal/-/dns-equal-1.0.0.tgz", + "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==", + "dev": true + }, + "dns-packet": { + "version": "1.3.4", + "resolved": "https://registry.npmmirror.com/dns-packet/-/dns-packet-1.3.4.tgz", + "integrity": "sha512-BQ6F4vycLXBvdrJZ6S3gZewt6rcrks9KBgM9vrhW+knGRqc8uEdT7fuCwloc7nny5xNoMJ17HGH0R/6fpo8ECA==", + "dev": true, + "requires": { + "ip": "^1.1.0", + "safe-buffer": "^5.0.1" + } + }, + "dns-txt": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/dns-txt/-/dns-txt-2.0.2.tgz", + "integrity": "sha512-Ix5PrWjphuSoUXV/Zv5gaFHjnaJtb02F2+Si3Ht9dyJ87+Z/lMmy+dpNHtTGraNK958ndXq2i+GLkWsWHcKaBQ==", + "dev": true, + "requires": { + "buffer-indexof": "^1.0.0" + } + }, + "doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "dom-converter": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/dom-converter/-/dom-converter-0.2.0.tgz", + "integrity": "sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==", + "dev": true, + "requires": { + "utila": "~0.4" + } + }, + "dom-serializer": { + "version": "0.2.2", + "resolved": "https://registry.npmmirror.com/dom-serializer/-/dom-serializer-0.2.2.tgz", + "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==", + "requires": { + "domelementtype": "^2.0.1", + "entities": "^2.0.0" + }, + "dependencies": { + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" + } + } + }, + "dom-to-image": { + "version": "2.6.0", + "resolved": "https://registry.npmmirror.com/dom-to-image/-/dom-to-image-2.6.0.tgz", + "integrity": "sha512-Dt0QdaHmLpjURjU7Tnu3AgYSF2LuOmksSGsUcE6ItvJoCWTBEmiMXcqBdNSAm9+QbbwD7JMoVsuuKX6ZVQv1qA==" + }, + "domain-browser": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/domain-browser/-/domain-browser-1.2.0.tgz", + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==" + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmmirror.com/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "dev": true, + "requires": { + "domelementtype": "^2.2.0" + }, + "dependencies": { + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true + } + } + }, + "domready": { + "version": "1.0.8", + "resolved": "https://registry.npmmirror.com/domready/-/domready-1.0.8.tgz", + "integrity": "sha512-uIzsOJUNk+AdGE9a6VDeessoMCzF8RrZvJCX/W8QtyfgdR6Uofn/MvRonih3OtCO79b2VDzDOymuiABrQ4z3XA==", + "dev": true + }, + "domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmmirror.com/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "dot-prop": { + "version": "5.3.0", + "resolved": "https://registry.npmmirror.com/dot-prop/-/dot-prop-5.3.0.tgz", + "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", + "dev": true, + "requires": { + "is-obj": "^2.0.0" + } + }, + "dotenv": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/dotenv/-/dotenv-7.0.0.tgz", + "integrity": "sha512-M3NhsLbV1i6HuGzBUH8vXrtxOk+tWmzWKDMbAVSUp3Zsjm7ywFeuwrUXhmhQyRK1q5B5GGy7hcXPbj3bnfZg2g==", + "dev": true + }, + "dotenv-expand": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz", + "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==", + "dev": true + }, + "duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "dev": true + }, + "duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmmirror.com/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "requires": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, + "easy-stack": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/easy-stack/-/easy-stack-1.0.1.tgz", + "integrity": "sha512-wK2sCs4feiiJeFXn3zvY0p41mdU5VUgbgs1rNsc/y5ngFUijdWd+iIN8eoyuZHKB8xN6BL4PdWmzqFmxNg6V2w==", + "dev": true + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", + "dev": true, + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "echarts": { + "version": "4.9.0", + "resolved": "https://registry.npmmirror.com/echarts/-/echarts-4.9.0.tgz", + "integrity": "sha512-+ugizgtJ+KmsJyyDPxaw2Br5FqzuBnyOWwcxPKO6y0gc5caYcfnEUIlNStx02necw8jmKmTafmpHhGo4XDtEIA==", + "requires": { + "zrender": "4.3.2" + } + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true + }, + "ejs": { + "version": "2.7.4", + "resolved": "https://registry.npmmirror.com/ejs/-/ejs-2.7.4.tgz", + "integrity": "sha512-7vmuyh5+kuUyJKePhQfRQBhXV5Ce+RnaeeQArKu1EAMpL3WbgMt5WG6uQZpEVvYSSsxMXRKOewtDk9RaTKXRlA==", + "dev": true + }, + "electron-to-chromium": { + "version": "1.4.121", + "resolved": "https://registry.npmmirror.com/electron-to-chromium/-/electron-to-chromium-1.4.121.tgz", + "integrity": "sha512-N7OXhMr1p2oa9EkOhmHpmOm43DHzs55dep2FF6M7y6px5QJBheqEE3nwwZ+xJowlff+AEmMOdg3ARYGB+0kzbA==", + "dev": true + }, + "elegant-spinner": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/elegant-spinner/-/elegant-spinner-1.0.1.tgz", + "integrity": "sha512-B+ZM+RXvRqQaAmkMlO/oSe5nMUOaUnyfGYCEHoR8wrXsZR2mA0XVibsxV1bvTwxdRWah1PkQqso2EzhILGHtEQ==", + "dev": true + }, + "element-resize-detector": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/element-resize-detector/-/element-resize-detector-1.2.4.tgz", + "integrity": "sha512-Fl5Ftk6WwXE0wqCgNoseKWndjzZlDCwuPTcoVZfCP9R3EHQF8qUtr3YUPNETegRBOKqQKPW3n4kiIWngGi8tKg==", + "requires": { + "batch-processor": "1.0.0" + } + }, + "elliptic": { + "version": "6.5.4", + "resolved": "https://registry.npmmirror.com/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", + "requires": { + "bn.js": "^4.11.9", + "brorand": "^1.1.0", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmmirror.com/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true + }, + "encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmmirror.com/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "dev": true, + "optional": true, + "requires": { + "iconv-lite": "^0.6.2" + }, + "dependencies": { + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "optional": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + } + } + }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmmirror.com/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, + "enhanced-resolve": { + "version": "4.5.0", + "resolved": "https://registry.npmmirror.com/enhanced-resolve/-/enhanced-resolve-4.5.0.tgz", + "integrity": "sha512-Nv9m36S/vxpsI+Hc4/ZGRs0n9mXqSWGGq49zxb/cJfPAQMbUtttJAlNPS4AQzaBdw/pKskw5bMbekT/Y7W/Wlg==", + "requires": { + "graceful-fs": "^4.1.2", + "memory-fs": "^0.5.0", + "tapable": "^1.0.0" + }, + "dependencies": { + "memory-fs": { + "version": "0.5.0", + "resolved": "https://registry.npmmirror.com/memory-fs/-/memory-fs-0.5.0.tgz", + "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==", + "requires": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + } + } + } + }, + "entities": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==" + }, + "env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "dev": true + }, + "envinfo": { + "version": "7.8.1", + "resolved": "https://registry.npmmirror.com/envinfo/-/envinfo-7.8.1.tgz", + "integrity": "sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==", + "dev": true + }, + "err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "dev": true + }, + "errno": { + "version": "0.1.8", + "resolved": "https://registry.npmmirror.com/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "requires": { + "prr": "~1.0.1" + } + }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "error-stack-parser": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/error-stack-parser/-/error-stack-parser-2.0.7.tgz", + "integrity": "sha512-chLOW0ZGRf4s8raLrDxa5sdkvPec5YdvwbFnqJme4rk0rFajP8mPtrDL1+I+CwrQDCjswDA5sREX7jYQDQs9vA==", + "dev": true, + "requires": { + "stackframe": "^1.1.1" + } + }, + "es-abstract": { + "version": "1.19.5", + "resolved": "https://registry.npmmirror.com/es-abstract/-/es-abstract-1.19.5.tgz", + "integrity": "sha512-Aa2G2+Rd3b6kxEUKTF4TaW67czBLyAv3z7VOhYRU50YBx+bbsYZ9xQP4lMNazePuFlybXI0V4MruPos7qUo5fA==", + "requires": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", + "get-symbol-description": "^1.0.0", + "has": "^1.0.3", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.4", + "is-negative-zero": "^2.0.2", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.2", + "is-string": "^1.0.7", + "is-weakref": "^1.0.2", + "object-inspect": "^1.12.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.1" + } + }, + "es-shim-unscopables": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz", + "integrity": "sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w==", + "dev": true, + "requires": { + "has": "^1.0.3" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + }, + "eslint": { + "version": "6.8.0", + "resolved": "https://registry.npmmirror.com/eslint/-/eslint-6.8.0.tgz", + "integrity": "sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.10.0", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^1.4.3", + "eslint-visitor-keys": "^1.1.0", + "espree": "^6.1.2", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^7.0.0", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.14", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.3", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^6.1.2", + "strip-ansi": "^5.2.0", + "strip-json-comments": "^3.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "dependencies": { + "acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmmirror.com/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true + }, + "acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmmirror.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true + }, + "ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmmirror.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "requires": { + "type-fest": "^0.21.3" + }, + "dependencies": { + "type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true + } + } + }, + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmmirror.com/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", + "dev": true + }, + "cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "requires": { + "restore-cursor": "^3.1.0" + } + }, + "cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "dev": true + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + }, + "espree": { + "version": "6.2.1", + "resolved": "https://registry.npmmirror.com/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "dev": true, + "requires": { + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" + } + }, + "external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dev": true, + "requires": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + } + }, + "figures": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dev": true, + "requires": { + "flat-cache": "^2.0.1" + } + }, + "flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "requires": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + } + }, + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmmirror.com/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmmirror.com/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "inquirer": { + "version": "7.3.3", + "resolved": "https://registry.npmmirror.com/inquirer/-/inquirer-7.3.3.tgz", + "integrity": "sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==", + "dev": true, + "requires": { + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.19", + "mute-stream": "0.0.8", + "run-async": "^2.4.0", + "rxjs": "^6.6.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6" + }, + "dependencies": { + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmmirror.com/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "dev": true + }, + "onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "dev": true + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "requires": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + } + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + } + } + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "dependencies": { + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, + "table": { + "version": "5.4.6", + "resolved": "https://registry.npmmirror.com/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "dev": true, + "requires": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } + }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true + }, + "write": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "requires": { + "mkdirp": "^0.5.1" + } + } + } + }, + "eslint-config-standard": { + "version": "12.0.0", + "resolved": "https://registry.npmmirror.com/eslint-config-standard/-/eslint-config-standard-12.0.0.tgz", + "integrity": "sha512-COUz8FnXhqFitYj4DTqHzidjIL/t4mumGZto5c7DrBpvWoie+Sn3P4sLEzUGeYhRElWuFEf8K1S1EfvD1vixCQ==", + "dev": true + }, + "eslint-import-resolver-node": { + "version": "0.3.6", + "resolved": "https://registry.npmmirror.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz", + "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==", + "dev": true, + "requires": { + "debug": "^3.2.7", + "resolve": "^1.20.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } + }, + "eslint-loader": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/eslint-loader/-/eslint-loader-2.2.1.tgz", + "integrity": "sha512-RLgV9hoCVsMLvOxCuNjdqOrUqIj9oJg8hF44vzJaYqsAHuY9G2YAeN3joQ9nxP0p5Th9iFSIpKo+SD8KISxXRg==", + "dev": true, + "requires": { + "loader-fs-cache": "^1.0.0", + "loader-utils": "^1.0.2", + "object-assign": "^4.0.1", + "object-hash": "^1.1.4", + "rimraf": "^2.6.1" + } + }, + "eslint-module-utils": { + "version": "2.7.3", + "resolved": "https://registry.npmmirror.com/eslint-module-utils/-/eslint-module-utils-2.7.3.tgz", + "integrity": "sha512-088JEC7O3lDZM9xGe0RerkOMd0EjFl+Yvd1jPWIkMT5u3H9+HC34mWWPnqPrN13gieT9pBOO+Qt07Nb/6TresQ==", + "dev": true, + "requires": { + "debug": "^3.2.7", + "find-up": "^2.1.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", + "dev": true + } + } + }, + "eslint-plugin-es": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/eslint-plugin-es/-/eslint-plugin-es-1.4.1.tgz", + "integrity": "sha512-5fa/gR2yR3NxQf+UXkeLeP8FBBl6tSgdrAz1+cF84v1FMM4twGwQoqTnn+QxFLcPOrF4pdKEJKDB/q9GoyJrCA==", + "dev": true, + "requires": { + "eslint-utils": "^1.4.2", + "regexpp": "^2.0.1" + }, + "dependencies": { + "regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "dev": true + } + } + }, + "eslint-plugin-import": { + "version": "2.26.0", + "resolved": "https://registry.npmmirror.com/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", + "dev": true, + "requires": { + "array-includes": "^3.1.4", + "array.prototype.flat": "^1.2.5", + "debug": "^2.6.9", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-module-utils": "^2.7.3", + "has": "^1.0.3", + "is-core-module": "^2.8.1", + "is-glob": "^4.0.3", + "minimatch": "^3.1.2", + "object.values": "^1.1.5", + "resolve": "^1.22.0", + "tsconfig-paths": "^3.14.1" + } + }, + "eslint-plugin-node": { + "version": "8.0.1", + "resolved": "https://registry.npmmirror.com/eslint-plugin-node/-/eslint-plugin-node-8.0.1.tgz", + "integrity": "sha512-ZjOjbjEi6jd82rIpFSgagv4CHWzG9xsQAVp1ZPlhRnnYxcTgENUVBvhYmkQ7GvT1QFijUSo69RaiOJKhMu6i8w==", + "dev": true, + "requires": { + "eslint-plugin-es": "^1.3.1", + "eslint-utils": "^1.3.1", + "ignore": "^5.0.2", + "minimatch": "^3.0.4", + "resolve": "^1.8.1", + "semver": "^5.5.0" + }, + "dependencies": { + "ignore": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "dev": true + } + } + }, + "eslint-plugin-promise": { + "version": "4.3.1", + "resolved": "https://registry.npmmirror.com/eslint-plugin-promise/-/eslint-plugin-promise-4.3.1.tgz", + "integrity": "sha512-bY2sGqyptzFBDLh/GMbAxfdJC+b0f23ME63FOE4+Jao0oZ3E1LEwFtWJX/1pGMJLiTtrSSern2CRM/g+dfc0eQ==", + "dev": true + }, + "eslint-plugin-standard": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz", + "integrity": "sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ==", + "dev": true + }, + "eslint-plugin-vue": { + "version": "6.2.2", + "resolved": "https://registry.npmmirror.com/eslint-plugin-vue/-/eslint-plugin-vue-6.2.2.tgz", + "integrity": "sha512-Nhc+oVAHm0uz/PkJAWscwIT4ijTrK5fqNqz9QB1D35SbbuMG1uB6Yr5AJpvPSWg+WOw7nYNswerYh0kOk64gqQ==", + "dev": true, + "requires": { + "natural-compare": "^1.4.0", + "semver": "^5.6.0", + "vue-eslint-parser": "^7.0.0" + }, + "dependencies": { + "acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmmirror.com/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true + }, + "acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmmirror.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + }, + "espree": { + "version": "6.2.1", + "resolved": "https://registry.npmmirror.com/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "dev": true, + "requires": { + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmmirror.com/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "vue-eslint-parser": { + "version": "7.11.0", + "resolved": "https://registry.npmmirror.com/vue-eslint-parser/-/vue-eslint-parser-7.11.0.tgz", + "integrity": "sha512-qh3VhDLeh773wjgNTl7ss0VejY9bMMa0GoDG2fQVyDzRFdiU3L7fw74tWZDHNQXdZqxO3EveQroa9ct39D2nqg==", + "dev": true, + "requires": { + "debug": "^4.1.1", + "eslint-scope": "^5.1.1", + "eslint-visitor-keys": "^1.1.0", + "espree": "^6.2.1", + "esquery": "^1.4.0", + "lodash": "^4.17.21", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + } + } + }, + "eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "eslint-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmmirror.com/eslint-utils/-/eslint-utils-1.4.3.tgz", + "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + } + }, + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + }, + "espree": { + "version": "3.5.4", + "resolved": "https://registry.npmmirror.com/espree/-/espree-3.5.4.tgz", + "integrity": "sha512-yAcIQxtmMiB/jL32dzEp2enBeidsB7xWPLNiw3IIkpVds1P+h7qF9YwJq1yUNzp2OKXgAprs4F61ih66UsoD1A==", + "dev": true, + "optional": true, + "requires": { + "acorn": "^5.5.0", + "acorn-jsx": "^3.0.0" + }, + "dependencies": { + "acorn": { + "version": "5.7.4", + "resolved": "https://registry.npmmirror.com/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", + "dev": true, + "optional": true + } + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, + "esquery": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/esquery/-/esquery-1.4.0.tgz", + "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "dev": true, + "requires": { + "estraverse": "^5.1.0" + }, + "dependencies": { + "estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmmirror.com/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true + } + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "requires": { + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmmirror.com/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" + } + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==" + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmmirror.com/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true + }, + "event-pubsub": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/event-pubsub/-/event-pubsub-4.3.0.tgz", + "integrity": "sha512-z7IyloorXvKbFx9Bpie2+vMJKKx1fH1EN5yiTfp8CiLOTptSYy1g8H4yDpGlEdshL1PBiFtBHepF2cNsqeEeFQ==", + "dev": true + }, + "eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmmirror.com/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, + "events": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + }, + "eventsource": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/eventsource/-/eventsource-1.1.0.tgz", + "integrity": "sha512-VSJjT5oCNrFvCS6igjzPAt5hBzQ2qPBFIbJ03zLI9SE0mxwZpMw6BfJrbFHm1a141AavMEB8JHmBhWAd66PfCg==", + "dev": true, + "requires": { + "original": "^1.0.0" + } + }, + "evp_bytestokey": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", + "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", + "requires": { + "md5.js": "^1.3.4", + "safe-buffer": "^5.1.1" + } + }, + "execa": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "dev": true, + "requires": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + }, + "expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmmirror.com/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", + "requires": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "expand-range": { + "version": "1.8.2", + "resolved": "https://registry.npmmirror.com/expand-range/-/expand-range-1.8.2.tgz", + "integrity": "sha512-AFASGfIlnIbkKPQwX1yHaDjFvh/1gyKJODme52V6IORh69uEYgZp0o9C+qsIGNVEiuuhQU0CSSl++Rlegg1qvA==", + "dev": true, + "requires": { + "fill-range": "^2.1.0" + }, + "dependencies": { + "fill-range": { + "version": "2.2.4", + "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-2.2.4.tgz", + "integrity": "sha512-cnrcCbj01+j2gTG921VZPnHbjmdAf8oQV/iGeV2kZxGSyfYjjTyY79ErsK1WJWMpw6DaApEX72binqJE+/d+5Q==", + "dev": true, + "requires": { + "is-number": "^2.1.0", + "isobject": "^2.0.0", + "randomatic": "^3.0.0", + "repeat-element": "^1.1.2", + "repeat-string": "^1.5.2" + } + }, + "is-number": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-number/-/is-number-2.1.0.tgz", + "integrity": "sha512-QUzH43Gfb9+5yckcrSA0VBDwEtDUchrk4F6tfJZQuNzDJbEDB9cZNzSfXGQ1jqmdDY/kl41lUOWM9syA8z8jlg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + } + }, + "isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "dev": true, + "requires": { + "isarray": "1.0.0" + } + }, + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "express": { + "version": "4.18.0", + "resolved": "https://registry.npmmirror.com/express/-/express-4.18.0.tgz", + "integrity": "sha512-EJEXxiTQJS3lIPrU1AE2vRuT7X7E+0KBbpm5GSoK524yl0K8X+er8zS2P14E64eqsVNoWbMCT7MpmQ+ErAhgRg==", + "dev": true, + "requires": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.0", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.10.3", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "qs": { + "version": "6.10.3", + "resolved": "https://registry.npmmirror.com/qs/-/qs-6.10.3.tgz", + "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "dev": true, + "requires": { + "side-channel": "^1.0.4" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + } + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, + "extend-shallow": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-3.0.2.tgz", + "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==", + "requires": { + "assign-symbols": "^1.0.0", + "is-extendable": "^1.0.1" + }, + "dependencies": { + "is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "requires": { + "is-plain-object": "^2.0.4" + } + } + } + }, + "external-editor": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/external-editor/-/external-editor-2.2.0.tgz", + "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", + "dev": true, + "optional": true, + "requires": { + "chardet": "^0.4.0", + "iconv-lite": "^0.4.17", + "tmp": "^0.0.33" + } + }, + "extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "requires": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "requires": { + "is-descriptor": "^1.0.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "requires": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + } + } + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==", + "dev": true + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "fast-glob": { + "version": "2.2.7", + "resolved": "https://registry.npmmirror.com/fast-glob/-/fast-glob-2.2.7.tgz", + "integrity": "sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw==", + "dev": true, + "requires": { + "@mrmlnc/readdir-enhanced": "^2.2.1", + "@nodelib/fs.stat": "^1.1.2", + "glob-parent": "^3.1.0", + "is-glob": "^4.0.0", + "merge2": "^1.2.3", + "micromatch": "^3.1.10" + }, + "dependencies": { + "glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==", + "dev": true, + "requires": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + }, + "dependencies": { + "is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==", + "dev": true, + "requires": { + "is-extglob": "^2.1.0" + } + } + } + } + } + }, + "fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmmirror.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true + }, + "fastparse": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/fastparse/-/fastparse-1.1.2.tgz", + "integrity": "sha512-483XLLxTVIwWK3QTrMGRqUfUpoOs/0hbQrl2oz4J0pAcm3A3bu84wxTFqGqkJzewCLdME38xJLJAxBABfQT8sQ==", + "dev": true + }, + "fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmmirror.com/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "dev": true, + "requires": { + "reusify": "^1.0.4" + } + }, + "faye-websocket": { + "version": "0.11.4", + "resolved": "https://registry.npmmirror.com/faye-websocket/-/faye-websocket-0.11.4.tgz", + "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", + "dev": true, + "requires": { + "websocket-driver": ">=0.5.1" + } + }, + "figgy-pudding": { + "version": "3.5.2", + "resolved": "https://registry.npmmirror.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz", + "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==" + }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/figures/-/figures-2.0.0.tgz", + "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "file-entry-cache": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/file-entry-cache/-/file-entry-cache-2.0.0.tgz", + "integrity": "sha512-uXP/zGzxxFvFfcZGgBIwotm+Tdc55ddPAzF7iHshP4YGaXMww7rSF9peD9D1sui5ebONg5UobsZv+FfgEpGv/w==", + "dev": true, + "optional": true, + "requires": { + "flat-cache": "^1.2.1", + "object-assign": "^4.0.1" + } + }, + "file-loader": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/file-loader/-/file-loader-3.0.1.tgz", + "integrity": "sha512-4sNIOXgtH/9WZq4NvlfU3Opn5ynUsqBwSLyM+I7UOwdGigTBYfVVQEwe/msZNX/j4pCJTIM14Fsw66Svo1oVrw==", + "dev": true, + "requires": { + "loader-utils": "^1.0.2", + "schema-utils": "^1.0.0" + } + }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "optional": true + }, + "filemanager-webpack-plugin": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/filemanager-webpack-plugin/-/filemanager-webpack-plugin-2.0.5.tgz", + "integrity": "sha512-Yj5XIdKI2AN2r66uZc4MZ/n18SMqe2KKlkAqHHMW1OwveDs2Vc5129CpbFcI73rq/rjqso+2HsxieS7u5sx6XA==", + "dev": true, + "requires": { + "archiver": "^3.0.0", + "cpx": "^1.5.0", + "fs-extra": "^7.0.0", + "make-dir": "^1.1.0", + "mv": "^2.1.1", + "rimraf": "^2.6.2" + }, + "dependencies": { + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true + } + } + }, + "filename-regex": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/filename-regex/-/filename-regex-2.0.1.tgz", + "integrity": "sha512-BTCqyBaWBTsauvnHiE8i562+EdJj+oUpkqWp2R1iCoR8f6oo8STRu3of7WJJ0TqWtxN50a5YFpzYK4Jj9esYfQ==", + "dev": true + }, + "filesize": { + "version": "3.6.1", + "resolved": "https://registry.npmmirror.com/filesize/-/filesize-3.6.1.tgz", + "integrity": "sha512-7KjR1vv6qnicaPMi1iiTcI85CyYwRO/PSFCu6SvqL8jN2Wjt/NIYQTFtFs7fSDCYOstUkEWIQGFUg5YZQfjlcg==", + "dev": true + }, + "fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==", + "requires": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "filter-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/filter-obj/-/filter-obj-1.1.0.tgz", + "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==", + "dev": true + }, + "finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dev": true, + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + } + }, + "find-babel-config": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/find-babel-config/-/find-babel-config-1.2.0.tgz", + "integrity": "sha512-jB2CHJeqy6a820ssiqwrKMeyC6nNdmrcgkKWJWmpoxpE8RKciYJXCcXRq1h2AzCo5I5BJeN2tkGEO3hLTuePRA==", + "dev": true, + "requires": { + "json5": "^0.5.1", + "path-exists": "^3.0.0" + }, + "dependencies": { + "json5": { + "version": "0.5.1", + "resolved": "https://registry.npmmirror.com/json5/-/json5-0.5.1.tgz", + "integrity": "sha512-4xrs1aW+6N5DalkqSVA8fxh458CXvR99WU8WLKmq4v8eWAL86Xo3BVqyd3SkA9wEVjCMqyvvRRkshAdOnBp5rw==", + "dev": true + } + } + }, + "find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "requires": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + } + }, + "find-index": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/find-index/-/find-index-0.1.1.tgz", + "integrity": "sha512-uJ5vWrfBKMcE6y2Z8834dwEZj9mNGxYa3t3I53OwFeuZ8D9oc2E5zcsrkuhX6h4iYrjhiv0T3szQmxlAV9uxDg==", + "dev": true + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "requires": { + "locate-path": "^3.0.0" + } + }, + "find-yarn-workspace-root": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/find-yarn-workspace-root/-/find-yarn-workspace-root-1.2.1.tgz", + "integrity": "sha512-dVtfb0WuQG+8Ag2uWkbG79hOUzEsRrhBzgfn86g2sJPkzmcpGdghbNTfUKGTxymFrY/tLIodDzLoW9nOJ4FY8Q==", + "dev": true, + "requires": { + "fs-extra": "^4.0.3", + "micromatch": "^3.1.4" + }, + "dependencies": { + "fs-extra": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-4.0.3.tgz", + "integrity": "sha512-q6rbdDd1o2mAnQreO7YADIxf/Whx4AHBiRf6d+/cVT8h44ss+lHgxf1FemcqDnQt9X3ct4McHr+JMGlYSsK7Cg==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + } + } + }, + "flat-cache": { + "version": "1.3.4", + "resolved": "https://registry.npmmirror.com/flat-cache/-/flat-cache-1.3.4.tgz", + "integrity": "sha512-VwyB3Lkgacfik2vhqR4uv2rvebqmDvFu4jlN/C1RzWoJEo8I7z4Q404oiqYCkq41mni8EzQnm95emU9seckwtg==", + "dev": true, + "optional": true, + "requires": { + "circular-json": "^0.3.1", + "graceful-fs": "^4.1.2", + "rimraf": "~2.6.2", + "write": "^0.2.1" + }, + "dependencies": { + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "optional": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "dev": true + }, + "flush-write-stream": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz", + "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==", + "requires": { + "inherits": "^2.0.3", + "readable-stream": "^2.3.6" + } + }, + "fn-name": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/fn-name/-/fn-name-2.0.1.tgz", + "integrity": "sha512-oIDB1rXf3BUnn00bh2jVM0byuqr94rBh6g7ZfdKcbmp1we2GQtPzKdloyvBXHs+q3fvxB8EqX5ecFba3RwCSjA==", + "dev": true + }, + "follow-redirects": { + "version": "1.14.9", + "resolved": "https://registry.npmmirror.com/follow-redirects/-/follow-redirects-1.14.9.tgz", + "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==" + }, + "for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==" + }, + "for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmmirror.com/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==", + "dev": true, + "requires": { + "for-in": "^1.0.1" + } + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==", + "dev": true + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmmirror.com/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true + }, + "fragment-cache": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/fragment-cache/-/fragment-cache-0.2.1.tgz", + "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==", + "requires": { + "map-cache": "^0.2.2" + } + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmmirror.com/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true + }, + "from2": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/from2/-/from2-2.3.0.tgz", + "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==", + "requires": { + "inherits": "^2.0.1", + "readable-stream": "^2.0.0" + } + }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true + }, + "fs-extra": { + "version": "7.0.1", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-7.0.1.tgz", + "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "fs-minipass": { + "version": "1.2.7", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", + "dev": true, + "requires": { + "minipass": "^2.6.0" + } + }, + "fs-write-stream-atomic": { + "version": "1.0.10", + "resolved": "https://registry.npmmirror.com/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz", + "integrity": "sha512-gehEzmPn2nAwr39eay+x3X34Ra+M2QlVUTLhkXPjWdeO8RF9kszk116avgBJM3ZyNHgHXBNx+VmPaFC36k0PzA==", + "requires": { + "graceful-fs": "^4.1.2", + "iferr": "^0.1.5", + "imurmurhash": "^0.1.4", + "readable-stream": "1 || 2" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "optional": true + }, + "fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmmirror.com/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + } + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", + "dev": true + }, + "functions-have-names": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/functions-have-names/-/functions-have-names-1.2.3.tgz", + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "dev": true + }, + "g-status": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/g-status/-/g-status-2.0.2.tgz", + "integrity": "sha512-kQoE9qH+T1AHKgSSD0Hkv98bobE90ILQcXAF4wvGgsr7uFqNvwmh8j+Lq3l0RVt3E3HjSbv2B9biEGcEtpHLCA==", + "dev": true, + "requires": { + "arrify": "^1.0.1", + "matcher": "^1.0.0", + "simple-git": "^1.85.0" + }, + "dependencies": { + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", + "dev": true + } + } + }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmmirror.com/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg==", + "dev": true, + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==", + "dev": true, + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==", + "dev": true, + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + } + } + }, + "gaze": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/gaze/-/gaze-1.1.3.tgz", + "integrity": "sha512-BRdNm8hbWzFzWHERTrejLqwHDfS4GibPoq5wjTPIoJHoBtKGPg3xAFfxmM+9ztbXelxcf2hwQcaz1PtmFeue8g==", + "dev": true, + "requires": { + "globule": "^1.0.0" + } + }, + "gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmmirror.com/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, + "get-own-enumerable-property-symbols": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz", + "integrity": "sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==", + "dev": true + }, + "get-pkg-repo": { + "version": "4.2.1", + "resolved": "https://registry.npmmirror.com/get-pkg-repo/-/get-pkg-repo-4.2.1.tgz", + "integrity": "sha512-2+QbHjFRfGB74v/pYWjd5OhU3TDIC2Gv/YKUTk/tCvAz0pkn/Mz6P3uByuBimLOcPvN2jYdScl3xGFSrx0jEcA==", + "dev": true, + "requires": { + "@hutson/parse-repository-url": "^3.0.0", + "hosted-git-info": "^4.0.0", + "through2": "^2.0.0", + "yargs": "^16.2.0" + }, + "dependencies": { + "hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "dev": true + }, + "get-stdin": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/get-stdin/-/get-stdin-6.0.0.tgz", + "integrity": "sha512-jp4tHawyV7+fkkSKyvjuLZswblUtz+SQKzSWnBbii16BuZksJlU1wuBYXY75r+duh/llF1ur6oNwi+2ZzjKZ7g==", + "dev": true + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, + "get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmmirror.com/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==" + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmmirror.com/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "git-raw-commits": { + "version": "2.0.11", + "resolved": "https://registry.npmmirror.com/git-raw-commits/-/git-raw-commits-2.0.11.tgz", + "integrity": "sha512-VnctFhw+xfj8Va1xtfEqCUD2XDrbAPSJx+hSrE5K7fGdjZruW7XV+QOrN7LF/RJyvspRiD2I0asWsxFp0ya26A==", + "dev": true, + "requires": { + "dargs": "^7.0.0", + "lodash": "^4.17.15", + "meow": "^8.0.0", + "split2": "^3.0.0", + "through2": "^4.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "through2": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dev": true, + "requires": { + "readable-stream": "3" + } + } + } + }, + "git-remote-origin-url": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/git-remote-origin-url/-/git-remote-origin-url-2.0.0.tgz", + "integrity": "sha512-eU+GGrZgccNJcsDH5LkXR3PB9M958hxc7sbA8DFJjrv9j4L2P/eZfKhM+QD6wyzpiv+b1BpK0XrYCxkovtjSLw==", + "dev": true, + "requires": { + "gitconfiglocal": "^1.0.0", + "pify": "^2.3.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true + } + } + }, + "git-semver-tags": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/git-semver-tags/-/git-semver-tags-4.1.1.tgz", + "integrity": "sha512-OWyMt5zBe7xFs8vglMmhM9lRQzCWL3WjHtxNNfJTMngGym7pC1kh8sP6jevfydJ6LP3ZvGxfb6ABYgPUM0mtsA==", + "dev": true, + "requires": { + "meow": "^8.0.0", + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "git-up": { + "version": "4.0.5", + "resolved": "https://registry.npmmirror.com/git-up/-/git-up-4.0.5.tgz", + "integrity": "sha512-YUvVDg/vX3d0syBsk/CKUTib0srcQME0JyHkL5BaYdwLsiCslPWmDSi8PUMo9pXYjrryMcmsCoCgsTpSCJEQaA==", + "dev": true, + "requires": { + "is-ssh": "^1.3.0", + "parse-url": "^6.0.0" + } + }, + "git-url-parse": { + "version": "11.6.0", + "resolved": "https://registry.npmmirror.com/git-url-parse/-/git-url-parse-11.6.0.tgz", + "integrity": "sha512-WWUxvJs5HsyHL6L08wOusa/IXYtMuCAhrMmnTjQPpBU0TTHyDhnOATNH3xNQz7YOQUsqIIPTGr4xiVti1Hsk5g==", + "dev": true, + "requires": { + "git-up": "^4.0.0" + } + }, + "gitconfiglocal": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz", + "integrity": "sha512-spLUXeTAVHxDtKsJc8FkFVgFtMdEN9qPGpL23VfSHx4fP4+Ds097IXLvymbnDH8FnmxX5Nr9bPw3A+AQ6mWEaQ==", + "dev": true, + "requires": { + "ini": "^1.3.2" + } + }, + "gl-matrix": { + "version": "3.4.3", + "resolved": "https://registry.npmmirror.com/gl-matrix/-/gl-matrix-3.4.3.tgz", + "integrity": "sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==" + }, + "glob": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-base": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/glob-base/-/glob-base-0.3.0.tgz", + "integrity": "sha512-ab1S1g1EbO7YzauaJLkgLp7DZVAqj9M/dvKlTt8DkXA2tiOIcSMrlVI2J1RZyB5iJVccEscjGn+kpOG9788MHA==", + "dev": true, + "requires": { + "glob-parent": "^2.0.0", + "is-glob": "^2.0.0" + }, + "dependencies": { + "glob-parent": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-2.0.0.tgz", + "integrity": "sha512-JDYOvfxio/t42HKdxkAYaCiBN7oYiuxykOxKxdaUW5Qn0zaYN3gRQWolrwdnf0shM9/EP0ebuuTmyoXNr1cC5w==", + "dev": true, + "requires": { + "is-glob": "^2.0.0" + } + }, + "is-extglob": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-extglob/-/is-extglob-1.0.0.tgz", + "integrity": "sha512-7Q+VbVafe6x2T+Tu6NcOf6sRklazEPmBoB3IWk3WdGZM2iGUwU/Oe3Wtq5lSEkDTTlpp8yx+5t4pzO/i9Ty1ww==", + "dev": true + }, + "is-glob": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-2.0.1.tgz", + "integrity": "sha512-a1dBeB19NXsf/E0+FHqkagizel/LQw2DjSQpvQrj3zT+jYPpaUCryPnrQajXKFLCMuf4I6FhRpaGtw4lPrG6Eg==", + "dev": true, + "requires": { + "is-extglob": "^1.0.0" + } + } + } + }, + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "requires": { + "is-glob": "^4.0.1" + } + }, + "glob-to-regexp": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz", + "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==", + "dev": true + }, + "glob2base": { + "version": "0.0.12", + "resolved": "https://registry.npmmirror.com/glob2base/-/glob2base-0.0.12.tgz", + "integrity": "sha512-ZyqlgowMbfj2NPjxaZZ/EtsXlOch28FRXgMd64vqZWk1bT9+wvSRLYD1om9M7QfQru51zJPAT17qXm4/zd+9QA==", + "dev": true, + "requires": { + "find-index": "^0.1.1" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmmirror.com/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "globby": { + "version": "9.2.0", + "resolved": "https://registry.npmmirror.com/globby/-/globby-9.2.0.tgz", + "integrity": "sha512-ollPHROa5mcxDEkwg6bPt3QbEf4pDQSNtd6JPL1YvOvAo/7/0VAm9TccUeoTmarjPw4pfUthSCqcyfNB1I3ZSg==", + "dev": true, + "requires": { + "@types/glob": "^7.1.1", + "array-union": "^1.0.2", + "dir-glob": "^2.2.2", + "fast-glob": "^2.2.6", + "glob": "^7.1.3", + "ignore": "^4.0.3", + "pify": "^4.0.1", + "slash": "^2.0.0" + }, + "dependencies": { + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmmirror.com/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + } + } + }, + "globule": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/globule/-/globule-1.3.3.tgz", + "integrity": "sha512-mb1aYtDbIjTu4ShMB85m3UzjX9BVKe9WCzsnfMSZk+K5GpIbBOexgg4PPCt5eHDEG5/ZQAUX2Kct02zfiPLsKg==", + "dev": true, + "requires": { + "glob": "~7.1.1", + "lodash": "~4.17.10", + "minimatch": "~3.0.2" + }, + "dependencies": { + "glob": { + "version": "7.1.7", + "resolved": "https://registry.npmmirror.com/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "minimatch": { + "version": "3.0.8", + "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.0.8.tgz", + "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + } + } + }, + "graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + }, + "graphlib": { + "version": "2.1.8", + "resolved": "https://registry.npmmirror.com/graphlib/-/graphlib-2.1.8.tgz", + "integrity": "sha512-jcLLfkpoVGmH7/InMC/1hIvOPSUh38oJtGhvrOFGzioE1DZ+0YW16RgmOJhHiuWTvGiJQ9Z1Ik43JvkRPRvE+A==", + "requires": { + "lodash": "^4.17.15" + } + }, + "gzip-size": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/gzip-size/-/gzip-size-5.1.1.tgz", + "integrity": "sha512-FNHi6mmoHvs1mxZAds4PpdCS6QG8B4C1krxJsMutgxl5t3+GlRTzzI3NEkifXx2pVsOvJdOGSmIgDhQ55FwdPA==", + "dev": true, + "requires": { + "duplexer": "^0.1.1", + "pify": "^4.0.1" + } + }, + "handle-thing": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/handle-thing/-/handle-thing-2.0.1.tgz", + "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", + "dev": true + }, + "handlebars": { + "version": "4.7.7", + "resolved": "https://registry.npmmirror.com/handlebars/-/handlebars-4.7.7.tgz", + "integrity": "sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==", + "dev": true, + "requires": { + "minimist": "^1.2.5", + "neo-async": "^2.6.0", + "source-map": "^0.6.1", + "uglify-js": "^3.1.4", + "wordwrap": "^1.0.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==", + "dev": true + }, + "har-validator": { + "version": "5.1.5", + "resolved": "https://registry.npmmirror.com/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "dev": true, + "requires": { + "ajv": "^6.12.3", + "har-schema": "^2.0.0" + } + }, + "hard-rejection": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/hard-rejection/-/hard-rejection-2.1.0.tgz", + "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", + "dev": true + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + } + } + }, + "has-bigints": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==" + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" + }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "requires": { + "get-intrinsic": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "requires": { + "has-symbols": "^1.0.2" + } + }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "dev": true + }, + "has-value": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/has-value/-/has-value-1.0.0.tgz", + "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==", + "requires": { + "get-value": "^2.0.6", + "has-values": "^1.0.0", + "isobject": "^3.0.0" + } + }, + "has-values": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/has-values/-/has-values-1.0.0.tgz", + "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==", + "requires": { + "is-number": "^3.0.0", + "kind-of": "^4.0.0" + }, + "dependencies": { + "kind-of": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-4.0.0.tgz", + "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "hash-base": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/hash-base/-/hash-base-3.1.0.tgz", + "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } + } + }, + "hash-sum": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/hash-sum/-/hash-sum-1.0.2.tgz", + "integrity": "sha512-fUs4B4L+mlt8/XAtSOGMUO1TXmAelItBPtJG7CyHJfYTdDjwisntGO2JQz7oUsatOY9o68+57eziUVNw/mRHmA==", + "dev": true + }, + "hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmmirror.com/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "requires": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "he": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true + }, + "hex-color-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz", + "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==", + "dev": true + }, + "hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==", + "requires": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "hoopy": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/hoopy/-/hoopy-0.1.4.tgz", + "integrity": "sha512-HRcs+2mr52W0K+x8RzcLzuPPmVIKMSv97RGHy0Ea9y/mpcaK+xTrjICA04KAHi4GRzxliNqNJEFYWHghy3rSfQ==", + "dev": true + }, + "hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", + "dev": true + }, + "hpack.js": { + "version": "2.1.6", + "resolved": "https://registry.npmmirror.com/hpack.js/-/hpack.js-2.1.6.tgz", + "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==", + "dev": true, + "requires": { + "inherits": "^2.0.1", + "obuf": "^1.0.0", + "readable-stream": "^2.0.1", + "wbuf": "^1.1.0" + } + }, + "hsl-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/hsl-regex/-/hsl-regex-1.0.0.tgz", + "integrity": "sha512-M5ezZw4LzXbBKMruP+BNANf0k+19hDQMgpzBIYnya//Al+fjNct9Wf3b1WedLqdEs2hKBvxq/jh+DsHJLj0F9A==", + "dev": true + }, + "hsla-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/hsla-regex/-/hsla-regex-1.0.0.tgz", + "integrity": "sha512-7Wn5GMLuHBjZCb2bTmnDOycho0p/7UVaAeqXZGbHrBCl6Yd/xDhQJAXe6Ga9AXJH2I5zY1dEdYw2u1UptnSBJA==", + "dev": true + }, + "html-entities": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/html-entities/-/html-entities-1.4.0.tgz", + "integrity": "sha512-8nxjcBcd8wovbeKx7h3wTji4e6+rhaVuPNpMqwWgnHh+N9ToqsCs6XztWRBPQ+UtzsoMAdKZtUENoVzU/EMtZA==", + "dev": true + }, + "html-minifier": { + "version": "3.5.21", + "resolved": "https://registry.npmmirror.com/html-minifier/-/html-minifier-3.5.21.tgz", + "integrity": "sha512-LKUKwuJDhxNa3uf/LPR/KVjm/l3rBqtYeCOAekvG8F1vItxMUpueGd94i/asDDr8/1u7InxzFA5EeGjhhG5mMA==", + "dev": true, + "requires": { + "camel-case": "3.0.x", + "clean-css": "4.2.x", + "commander": "2.17.x", + "he": "1.2.x", + "param-case": "2.1.x", + "relateurl": "0.2.x", + "uglify-js": "3.4.x" + }, + "dependencies": { + "commander": { + "version": "2.17.1", + "resolved": "https://registry.npmmirror.com/commander/-/commander-2.17.1.tgz", + "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==", + "dev": true + } + } + }, + "html-tags": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/html-tags/-/html-tags-2.0.0.tgz", + "integrity": "sha512-+Il6N8cCo2wB/Vd3gqy/8TZhTD3QvcVeQLCnZiGkGCH3JP28IgGAY41giccp2W4R3jfyJPAP318FQTa1yU7K7g==", + "dev": true + }, + "html-webpack-plugin": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "integrity": "sha512-Br4ifmjQojUP4EmHnRBoUIYcZ9J7M4bTMcm7u6xoIAIuq2Nte4TzXX0533owvkQKQD1WeMTTTyD4Ni4QKxS0Bg==", + "dev": true, + "requires": { + "html-minifier": "^3.2.3", + "loader-utils": "^0.2.16", + "lodash": "^4.17.3", + "pretty-error": "^2.0.2", + "tapable": "^1.0.0", + "toposort": "^1.0.0", + "util.promisify": "1.0.0" + }, + "dependencies": { + "big.js": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/big.js/-/big.js-3.2.0.tgz", + "integrity": "sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q==", + "dev": true + }, + "emojis-list": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/emojis-list/-/emojis-list-2.1.0.tgz", + "integrity": "sha512-knHEZMgs8BB+MInokmNTg/OyPlAddghe1YBgNwJBc5zsJi/uyIcXoSDsL/W9ymOsBoBGdPIHXYJ9+qKFwRwDng==", + "dev": true + }, + "json5": { + "version": "0.5.1", + "resolved": "https://registry.npmmirror.com/json5/-/json5-0.5.1.tgz", + "integrity": "sha512-4xrs1aW+6N5DalkqSVA8fxh458CXvR99WU8WLKmq4v8eWAL86Xo3BVqyd3SkA9wEVjCMqyvvRRkshAdOnBp5rw==", + "dev": true + }, + "loader-utils": { + "version": "0.2.17", + "resolved": "https://registry.npmmirror.com/loader-utils/-/loader-utils-0.2.17.tgz", + "integrity": "sha512-tiv66G0SmiOx+pLWMtGEkfSEejxvb6N6uRrQjfWJIT79W9GMpgKeCAmm9aVBKtd4WEgntciI8CsGqjpDoCWJug==", + "dev": true, + "requires": { + "big.js": "^3.1.3", + "emojis-list": "^2.0.0", + "json5": "^0.5.0", + "object-assign": "^4.0.1" + } + }, + "util.promisify": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/util.promisify/-/util.promisify-1.0.0.tgz", + "integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==", + "dev": true, + "requires": { + "define-properties": "^1.1.2", + "object.getownpropertydescriptors": "^2.0.3" + } + } + } + }, + "htmlparser2": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/htmlparser2/-/htmlparser2-6.1.0.tgz", + "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==", + "dev": true, + "requires": { + "domelementtype": "^2.0.1", + "domhandler": "^4.0.0", + "domutils": "^2.5.2", + "entities": "^2.0.0" + }, + "dependencies": { + "dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "dev": true, + "requires": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + } + }, + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true + }, + "domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmmirror.com/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "dev": true, + "requires": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + } + } + } + }, + "http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "dev": true + }, + "http-deceiver": { + "version": "1.2.7", + "resolved": "https://registry.npmmirror.com/http-deceiver/-/http-deceiver-1.2.7.tgz", + "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==", + "dev": true + }, + "http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "requires": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + } + }, + "http-parser-js": { + "version": "0.5.6", + "resolved": "https://registry.npmmirror.com/http-parser-js/-/http-parser-js-0.5.6.tgz", + "integrity": "sha512-vDlkRPDJn93swjcjqMSaGSPABbIarsr1TLAui/gLDXzV5VsJNdXNzMYDyNBLQkjWQCJ1uizu8T2oDMhmGt0PRA==", + "dev": true + }, + "http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmmirror.com/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dev": true, + "requires": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + } + }, + "http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "dev": true, + "requires": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "http-proxy-middleware": { + "version": "0.19.1", + "resolved": "https://registry.npmmirror.com/http-proxy-middleware/-/http-proxy-middleware-0.19.1.tgz", + "integrity": "sha512-yHYTgWMQO8VvwNS22eLLloAkvungsKdKTLO8AJlftYIKNfJr3GK3zK0ZCfzDDGUBttdGc8xFy1mCitvNKQtC3Q==", + "dev": true, + "requires": { + "http-proxy": "^1.17.0", + "is-glob": "^4.0.0", + "lodash": "^4.17.11", + "micromatch": "^3.1.10" + } + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", + "dev": true, + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "https-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/https-browserify/-/https-browserify-1.0.0.tgz", + "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==" + }, + "https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "requires": { + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "human-signals": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/human-signals/-/human-signals-1.1.1.tgz", + "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==", + "dev": true + }, + "humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dev": true, + "requires": { + "ms": "^2.0.0" + } + }, + "husky": { + "version": "1.3.1", + "resolved": "https://registry.npmmirror.com/husky/-/husky-1.3.1.tgz", + "integrity": "sha512-86U6sVVVf4b5NYSZ0yvv88dRgBSSXXmHaiq5pP4KDj5JVzdwKgBjEtUPOm8hcoytezFwbU+7gotXNhpHdystlg==", + "dev": true, + "requires": { + "cosmiconfig": "^5.0.7", + "execa": "^1.0.0", + "find-up": "^3.0.0", + "get-stdin": "^6.0.0", + "is-ci": "^2.0.0", + "pkg-dir": "^3.0.0", + "please-upgrade-node": "^3.1.1", + "read-pkg": "^4.0.1", + "run-node": "^1.0.0", + "slash": "^2.0.0" + }, + "dependencies": { + "ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "dev": true + }, + "is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dev": true, + "requires": { + "ci-info": "^2.0.0" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true + }, + "read-pkg": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/read-pkg/-/read-pkg-4.0.1.tgz", + "integrity": "sha512-+UBirHHDm5J+3WDmLBZYSklRYg82nMlz+enn+GMZ22nSR2f4bzxmhso6rzQW/3mT2PVzpzDTiYIZahk8UmZ44w==", + "dev": true, + "requires": { + "normalize-package-data": "^2.3.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0" + } + } + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "icss-replace-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz", + "integrity": "sha512-chIaY3Vh2mh2Q3RGXttaDIzeiPvaVXJ+C4DAh/w3c37SKZ/U6PGMmuicR2EQQp9bKG8zLMCl7I+PtIoOOPp8Gg==", + "dev": true + }, + "icss-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/icss-utils/-/icss-utils-2.1.0.tgz", + "integrity": "sha512-bsVoyn/1V4R1kYYjLcWLedozAM4FClZUdjE9nIr8uWY7xs78y9DATgwz2wGU7M+7z55KenmmTkN2DVJ7bqzjAA==", + "dev": true, + "requires": { + "postcss": "^6.0.1" + }, + "dependencies": { + "postcss": { + "version": "6.0.23", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-6.0.23.tgz", + "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==", + "dev": true, + "requires": { + "chalk": "^2.4.1", + "source-map": "^0.6.1", + "supports-color": "^5.4.0" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "iferr": { + "version": "0.1.5", + "resolved": "https://registry.npmmirror.com/iferr/-/iferr-0.1.5.tgz", + "integrity": "sha512-DUNFN5j7Tln0D+TxzloUjKB+CtVu6myn0JEFak6dG18mNt9YkQ6lzGCdafwofISZ1lLF3xRHJ98VKy9ynkcFaA==" + }, + "ignore": { + "version": "3.3.10", + "resolved": "https://registry.npmmirror.com/ignore/-/ignore-3.3.10.tgz", + "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==", + "dev": true + }, + "ignore-walk": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/ignore-walk/-/ignore-walk-3.0.4.tgz", + "integrity": "sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==", + "dev": true, + "requires": { + "minimatch": "^3.0.4" + } + }, + "image-size": { + "version": "0.5.5", + "resolved": "https://registry.npmmirror.com/image-size/-/image-size-0.5.5.tgz", + "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==", + "dev": true + }, + "import-cwd": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/import-cwd/-/import-cwd-2.1.0.tgz", + "integrity": "sha512-Ew5AZzJQFqrOV5BTW3EIoHAnoie1LojZLXKcCQ/yTRyVZosBhK1x1ViYjHGf5pAFOq8ZyChZp6m/fSN7pJyZtg==", + "dev": true, + "requires": { + "import-from": "^2.1.0" + } + }, + "import-fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/import-fresh/-/import-fresh-2.0.0.tgz", + "integrity": "sha512-eZ5H8rcgYazHbKC3PG4ClHNykCSxtAhxSSEM+2mb+7evD2CKF5V7c0dNum7AdpDh0ZdICwZY9sRSn8f+KH96sg==", + "dev": true, + "requires": { + "caller-path": "^2.0.0", + "resolve-from": "^3.0.0" + }, + "dependencies": { + "caller-path": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/caller-path/-/caller-path-2.0.0.tgz", + "integrity": "sha512-MCL3sf6nCSXOwCTzvPKhN18TU7AHTvdtam8DAogxcrJ8Rjfbbg7Lgng64H9Iy+vUV6VGFClN/TyxBkAebLRR4A==", + "dev": true, + "requires": { + "caller-callsite": "^2.0.0" + } + }, + "resolve-from": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-3.0.0.tgz", + "integrity": "sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw==", + "dev": true + } + } + }, + "import-from": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/import-from/-/import-from-2.1.0.tgz", + "integrity": "sha512-0vdnLL2wSGnhlRmzHJAg5JHjt1l2vYhzJ7tNLGbeVg0fse56tpGaH0uzH+r9Slej+BSXXEHvBKDEnVSLLE9/+w==", + "dev": true, + "requires": { + "resolve-from": "^3.0.0" + }, + "dependencies": { + "resolve-from": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-3.0.0.tgz", + "integrity": "sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw==", + "dev": true + } + } + }, + "import-local": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/import-local/-/import-local-2.0.0.tgz", + "integrity": "sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ==", + "dev": true, + "requires": { + "pkg-dir": "^3.0.0", + "resolve-cwd": "^2.0.0" + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==" + }, + "in-publish": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/in-publish/-/in-publish-2.0.1.tgz", + "integrity": "sha512-oDM0kUSNFC31ShNxHKUyfZKy8ZeXZBWMjMdZHKLOk13uvT27VTL/QzRGfRUcevJhpkZAvlhPYuXkF7eNWrtyxQ==", + "dev": true + }, + "indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true + }, + "indexes-of": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/indexes-of/-/indexes-of-1.0.1.tgz", + "integrity": "sha512-bup+4tap3Hympa+JBJUG7XuOsdNQ6fxt0MHyXMKuLBKn0OqsTfvUxkUrroEX1+B2VsSHvCjiIcZVxRtYa4nllA==", + "dev": true + }, + "infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==" + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmmirror.com/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true + }, + "init-package-json": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/init-package-json/-/init-package-json-2.0.5.tgz", + "integrity": "sha512-u1uGAtEFu3VA6HNl/yUWw57jmKEMx8SKOxHhxjGnOFUiIlFnohKDFg4ZrPpv9wWqk44nDxGJAtqjdQFm+9XXQA==", + "dev": true, + "requires": { + "npm-package-arg": "^8.1.5", + "promzard": "^0.3.0", + "read": "~1.0.1", + "read-package-json": "^4.1.1", + "semver": "^7.3.5", + "validate-npm-package-license": "^3.0.4", + "validate-npm-package-name": "^3.0.0" + }, + "dependencies": { + "hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "normalize-package-data": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", + "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", + "dev": true, + "requires": { + "hosted-git-info": "^4.0.1", + "is-core-module": "^2.5.0", + "semver": "^7.3.4", + "validate-npm-package-license": "^3.0.1" + } + }, + "read-package-json": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/read-package-json/-/read-package-json-4.1.2.tgz", + "integrity": "sha512-Dqer4pqzamDE2O4M55xp1qZMuLPqi4ldk2ya648FOMHRjwMzFhuxVrG04wd0c38IsvkVdr3vgHI6z+QTPdAjrQ==", + "dev": true, + "requires": { + "glob": "^7.1.1", + "json-parse-even-better-errors": "^2.3.0", + "normalize-package-data": "^3.0.0", + "npm-normalize-package-bin": "^1.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "inquirer": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/inquirer/-/inquirer-3.3.0.tgz", + "integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==", + "dev": true, + "optional": true, + "requires": { + "ansi-escapes": "^3.0.0", + "chalk": "^2.0.0", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^2.0.4", + "figures": "^2.0.0", + "lodash": "^4.3.0", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rx-lite": "^4.0.8", + "rx-lite-aggregates": "^4.0.8", + "string-width": "^2.1.0", + "strip-ansi": "^4.0.0", + "through": "^2.3.6" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", + "dev": true, + "optional": true + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "dev": true, + "optional": true, + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "internal-ip": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/internal-ip/-/internal-ip-4.3.0.tgz", + "integrity": "sha512-S1zBo1D6zcsyuC6PMmY5+55YMILQ9av8lotMx447Bq6SAgo/sDK6y6uUKmuYhW7eacnIhFfsPmCNYdDzsnnDCg==", + "dev": true, + "requires": { + "default-gateway": "^4.2.0", + "ipaddr.js": "^1.9.0" + }, + "dependencies": { + "default-gateway": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/default-gateway/-/default-gateway-4.2.0.tgz", + "integrity": "sha512-h6sMrVB1VMWVrW13mSc6ia/DwYYw5MN6+exNu1OaJeFac5aSAvwM7lZ0NVfTABuSkQelr4h5oebg3KB1XPdjgA==", + "dev": true, + "requires": { + "execa": "^1.0.0", + "ip-regex": "^2.1.0" + } + } + } + }, + "internal-slot": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/internal-slot/-/internal-slot-1.0.3.tgz", + "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", + "requires": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + } + }, + "invariant": { + "version": "2.2.4", + "resolved": "https://registry.npmmirror.com/invariant/-/invariant-2.2.4.tgz", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "dev": true, + "requires": { + "loose-envify": "^1.0.0" + } + }, + "ip": { + "version": "1.1.5", + "resolved": "https://registry.npmmirror.com/ip/-/ip-1.1.5.tgz", + "integrity": "sha512-rBtCAQAJm8A110nbwn6YdveUnuZH3WrC36IwkRXxDnq53JvXA2NVQvB7IHyKomxK1MJ4VDNw3UtFDdXQ+AvLYA==", + "dev": true + }, + "ip-regex": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/ip-regex/-/ip-regex-2.1.0.tgz", + "integrity": "sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==", + "dev": true + }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true + }, + "is-absolute-url": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-absolute-url/-/is-absolute-url-2.1.0.tgz", + "integrity": "sha512-vOx7VprsKyllwjSkLV79NIhpyLfr3jAp7VaTCMXOJHu4m0Ew1CZ2fcjASwmV1jI3BWuWHB013M48eyeldk9gYg==", + "dev": true + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmmirror.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-any-array": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-any-array/-/is-any-array-2.0.0.tgz", + "integrity": "sha512-WdPV58rT3aOWXvvyuBydnCq4S2BM1Yz8shKxlEpk/6x+GX202XRvXOycEFtNgnHVLoc46hpexPFx8Pz1/sMS0w==" + }, + "is-arguments": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/is-arguments/-/is-arguments-1.1.1.tgz", + "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true + }, + "is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "requires": { + "has-bigints": "^1.0.1" + } + }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "optional": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, + "is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmmirror.com/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, + "is-callable": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/is-callable/-/is-callable-1.2.4.tgz", + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==" + }, + "is-ci": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/is-ci/-/is-ci-1.2.1.tgz", + "integrity": "sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==", + "dev": true, + "requires": { + "ci-info": "^1.5.0" + } + }, + "is-color-stop": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/is-color-stop/-/is-color-stop-1.1.0.tgz", + "integrity": "sha512-H1U8Vz0cfXNujrJzEcvvwMDW9Ra+biSYA3ThdQvAnMLJkEHQXn6bWzLkxHtVYJ+Sdbx0b6finn3jZiaVe7MAHA==", + "dev": true, + "requires": { + "css-color-names": "^0.0.4", + "hex-color-regex": "^1.1.0", + "hsl-regex": "^1.0.0", + "hsla-regex": "^1.0.0", + "rgb-regex": "^1.0.1", + "rgba-regex": "^1.0.0" + } + }, + "is-core-module": { + "version": "2.9.0", + "resolved": "https://registry.npmmirror.com/is-core-module/-/is-core-module-2.9.0.tgz", + "integrity": "sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A==", + "dev": true, + "requires": { + "has": "^1.0.3" + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmmirror.com/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "dependencies": { + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==" + } + } + }, + "is-directory": { + "version": "0.3.1", + "resolved": "https://registry.npmmirror.com/is-directory/-/is-directory-0.3.1.tgz", + "integrity": "sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==", + "dev": true + }, + "is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "dev": true + }, + "is-dotfile": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/is-dotfile/-/is-dotfile-1.0.3.tgz", + "integrity": "sha512-9YclgOGtN/f8zx0Pr4FQYMdibBiTaH3sn52vjYip4ZSf6C4/6RfTEZ+MR4GvKhCxdPh21Bg42/WL55f6KSnKpg==", + "dev": true + }, + "is-equal-shallow": { + "version": "0.1.3", + "resolved": "https://registry.npmmirror.com/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz", + "integrity": "sha512-0EygVC5qPvIyb+gSz7zdD5/AAoS6Qrx1e//6N4yv4oNm30kqvdmG66oZFWVlQHUWe5OjP08FuTw2IdT0EOTcYA==", + "dev": true, + "requires": { + "is-primitive": "^2.0.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==" + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==" + }, + "is-finite": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/is-finite/-/is-finite-1.1.0.tgz", + "integrity": "sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, + "is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-lambda": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-lambda/-/is-lambda-1.0.1.tgz", + "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", + "dev": true + }, + "is-negative-zero": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==" + }, + "is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-number-object": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "dev": true + }, + "is-observable": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/is-observable/-/is-observable-1.1.0.tgz", + "integrity": "sha512-NqCa4Sa2d+u7BWc6CukaObG3Fh+CU9bvixbpcXYhy2VvYS7vVGIdAgnIS5Ks3A/cqk4rebLJ9s8zBstT2aKnIA==", + "dev": true, + "requires": { + "symbol-observable": "^1.1.0" + } + }, + "is-path-cwd": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", + "dev": true + }, + "is-path-in-cwd": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz", + "integrity": "sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==", + "dev": true, + "requires": { + "is-path-inside": "^2.1.0" + } + }, + "is-path-inside": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-path-inside/-/is-path-inside-2.1.0.tgz", + "integrity": "sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==", + "dev": true, + "requires": { + "path-is-inside": "^1.0.2" + } + }, + "is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", + "dev": true + }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "requires": { + "isobject": "^3.0.1" + } + }, + "is-posix-bracket": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz", + "integrity": "sha512-Yu68oeXJ7LeWNmZ3Zov/xg/oDBnBK2RNxwYY1ilNJX+tKKZqgPK+qOn/Gs9jEu66KDY9Netf5XLKNGzas/vPfQ==", + "dev": true + }, + "is-primitive": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-primitive/-/is-primitive-2.0.0.tgz", + "integrity": "sha512-N3w1tFaRfk3UrPfqeRyD+GYDASU3W5VinKhlORy8EWVf/sIdDL9GAcew85XmktCfH+ngG7SRXEVDoO18WMdB/Q==", + "dev": true + }, + "is-promise": { + "version": "2.2.2", + "resolved": "https://registry.npmmirror.com/is-promise/-/is-promise-2.2.2.tgz", + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==", + "dev": true + }, + "is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-regexp": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-regexp/-/is-regexp-1.0.0.tgz", + "integrity": "sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==", + "dev": true + }, + "is-resolvable": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/is-resolvable/-/is-resolvable-1.1.0.tgz", + "integrity": "sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg==", + "dev": true + }, + "is-shared-array-buffer": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "requires": { + "call-bind": "^1.0.2" + } + }, + "is-ssh": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/is-ssh/-/is-ssh-1.3.3.tgz", + "integrity": "sha512-NKzJmQzJfEEma3w5cJNcUMxoXfDjz0Zj0eyCalHn2E6VOwlzjZo0yuO2fcBSf8zhFuVCL/82/r5gRcoi6aEPVQ==", + "dev": true, + "requires": { + "protocols": "^1.1.0" + } + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "dev": true + }, + "is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "requires": { + "has-symbols": "^1.0.2" + } + }, + "is-text-path": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-text-path/-/is-text-path-1.0.1.tgz", + "integrity": "sha512-xFuJpne9oFz5qDaodwmmG08e3CawH/2ZV8Qqza1Ko7Sk8POWbkRdwIoAWVhqvq0XeUzANEhKo2n0IXUGBm7A/w==", + "dev": true, + "requires": { + "text-extensions": "^1.0.0" + } + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==", + "dev": true + }, + "is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==", + "dev": true + }, + "is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "requires": { + "call-bind": "^1.0.2" + } + }, + "is-what": { + "version": "3.14.1", + "resolved": "https://registry.npmmirror.com/is-what/-/is-what-3.14.1.tgz", + "integrity": "sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==", + "dev": true + }, + "is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==" + }, + "is-wsl": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/is-wsl/-/is-wsl-1.1.0.tgz", + "integrity": "sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==", + "dev": true + }, + "iview": { + "version": "3.5.4", + "resolved": "https://registry.npmmirror.com/iview/-/iview-3.5.4.tgz", + "integrity": "sha512-CEDHdAXxpGciMV+m1jdMDs0UVqzk/AaFhCDtSGcKDLidXRMOG2TgCZhtEHhI4mf3lchKEUFO/BcXtHAfBNuy7Q==", + "requires": { + "async-validator": "^1.12.2", + "deepmerge": "^2.2.1", + "element-resize-detector": "^1.2.0", + "js-calendar": "^1.2.3", + "lodash.throttle": "^4.1.1", + "popper.js": "^1.14.6", + "tinycolor2": "^1.4.1", + "v-click-outside-x": "^4.0.19" + } + }, + "javascript-stringify": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/javascript-stringify/-/javascript-stringify-1.6.0.tgz", + "integrity": "sha512-fnjC0up+0SjEJtgmmG+teeel68kutkvzfctO/KxE3qJlbunkJYAshgH3boU++gSBHP8z5/r0ts0qRIrHf0RTQQ==", + "dev": true + }, + "js-base64": { + "version": "2.6.4", + "resolved": "https://registry.npmmirror.com/js-base64/-/js-base64-2.6.4.tgz", + "integrity": "sha512-pZe//GGmwJndub7ZghVHz7vjb2LgC1m8B07Au3eYqeqv9emhESByMXxaEgkUkEqJe87oBbSniGYoQNIBklc7IQ==", + "dev": true + }, + "js-calendar": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/js-calendar/-/js-calendar-1.2.3.tgz", + "integrity": "sha512-dAA1/Zbp4+c5E+ARCVTIuKepXsNLzSYfzvOimiYD4S5eeP9QuplSHLcdhfqFSwyM1o1u6ku6RRRCyaZ0YAjiBw==" + }, + "js-levenshtein": { + "version": "1.1.6", + "resolved": "https://registry.npmmirror.com/js-levenshtein/-/js-levenshtein-1.1.6.tgz", + "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", + "dev": true + }, + "js-message": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/js-message/-/js-message-1.0.7.tgz", + "integrity": "sha512-efJLHhLjIyKRewNS9EGZ4UpI8NguuL6fKkhRxVuMmrGV2xN/0APGdQYwLFky5w9naebSZ0OwAGp0G6/2Cg90rA==", + "dev": true + }, + "js-queue": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/js-queue/-/js-queue-2.0.2.tgz", + "integrity": "sha512-pbKLsbCfi7kriM3s1J4DDCo7jQkI58zPLHi0heXPzPlj0hjUsm+FesPUbE0DSbIVIK503A36aUBoCN7eMFedkA==", + "dev": true, + "requires": { + "easy-stack": "^1.0.1" + } + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmmirror.com/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==", + "dev": true + }, + "jsencrypt": { + "version": "3.2.1", + "resolved": "https://registry.npmmirror.com/jsencrypt/-/jsencrypt-3.2.1.tgz", + "integrity": "sha512-k1sD5QV0KPn+D8uG9AdGzTQuamt82QZ3A3l6f7TRwMU6Oi2Vg0BsL+wZIQBONcraO1pc78ExMdvmBBJ8WhNYUA==" + }, + "jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmmirror.com/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true + }, + "json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==" + }, + "json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmmirror.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true + }, + "json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "dev": true + }, + "json5": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "requires": { + "minimist": "^1.2.0" + } + }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6" + } + }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmmirror.com/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", + "dev": true + }, + "jsprim": { + "version": "1.4.2", + "resolved": "https://registry.npmmirror.com/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.4.0", + "verror": "1.10.0" + } + }, + "killable": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/killable/-/killable-1.0.1.tgz", + "integrity": "sha512-LzqtLKlUwirEUyl/nicirVmNiPvYs7l5n8wOPP7fyJVpUPkvCnW/vuiXGpylGUlnPDnB7311rARzAt3Mhswpjg==", + "dev": true + }, + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + }, + "klaw-sync": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/klaw-sync/-/klaw-sync-6.0.0.tgz", + "integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11" + } + }, + "launch-editor": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/launch-editor/-/launch-editor-2.3.0.tgz", + "integrity": "sha512-3QrsCXejlWYHjBPFXTyGNhPj4rrQdB+5+r5r3wArpLH201aR+nWUgw/zKKkTmilCfY/sv6u8qo98pNvtg8LUTA==", + "dev": true, + "requires": { + "picocolors": "^1.0.0", + "shell-quote": "^1.6.1" + } + }, + "launch-editor-middleware": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/launch-editor-middleware/-/launch-editor-middleware-2.3.0.tgz", + "integrity": "sha512-GJR64trLdFFwCoL9DMn/d1SZX0OzTDPixu4mcfWTShQ4tIqCHCGvlg9fOEYQXyBlrSMQwylsJfUWncheShfV2w==", + "dev": true, + "requires": { + "launch-editor": "^2.3.0" + } + }, + "lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dev": true, + "requires": { + "readable-stream": "^2.0.5" + } + }, + "lerna": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/lerna/-/lerna-4.0.0.tgz", + "integrity": "sha512-DD/i1znurfOmNJb0OBw66NmNqiM8kF6uIrzrJ0wGE3VNdzeOhz9ziWLYiRaZDGGwgbcjOo6eIfcx9O5Qynz+kg==", + "dev": true, + "requires": { + "@lerna/add": "4.0.0", + "@lerna/bootstrap": "4.0.0", + "@lerna/changed": "4.0.0", + "@lerna/clean": "4.0.0", + "@lerna/cli": "4.0.0", + "@lerna/create": "4.0.0", + "@lerna/diff": "4.0.0", + "@lerna/exec": "4.0.0", + "@lerna/import": "4.0.0", + "@lerna/info": "4.0.0", + "@lerna/init": "4.0.0", + "@lerna/link": "4.0.0", + "@lerna/list": "4.0.0", + "@lerna/publish": "4.0.0", + "@lerna/run": "4.0.0", + "@lerna/version": "4.0.0", + "import-local": "^3.0.2", + "npmlog": "^4.1.2" + }, + "dependencies": { + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "import-local": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/import-local/-/import-local-3.1.0.tgz", + "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==", + "dev": true, + "requires": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "requires": { + "find-up": "^4.0.0" + } + }, + "resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "requires": { + "resolve-from": "^5.0.0" + } + }, + "resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true + } + } + }, + "less": { + "version": "3.13.1", + "resolved": "https://registry.npmmirror.com/less/-/less-3.13.1.tgz", + "integrity": "sha512-SwA1aQXGUvp+P5XdZslUOhhLnClSLIjWvJhmd+Vgib5BFIr9lMNlQwmwUNOjXThF/A0x+MCYYPeWEfeWiLRnTw==", + "dev": true, + "requires": { + "copy-anything": "^2.0.1", + "errno": "^0.1.1", + "graceful-fs": "^4.1.2", + "image-size": "~0.5.0", + "make-dir": "^2.1.0", + "mime": "^1.4.1", + "native-request": "^1.0.5", + "source-map": "~0.6.0", + "tslib": "^1.10.0" + }, + "dependencies": { + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "optional": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "optional": true + }, + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true + } + } + }, + "less-loader": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/less-loader/-/less-loader-6.1.0.tgz", + "integrity": "sha512-/jLzOwLyqJ7Kt3xg5sHHkXtOyShWwFj410K9Si9WO+/h8rmYxxkSR0A3/hFEntWudE20zZnWMtpMYnLzqTVdUA==", + "dev": true, + "requires": { + "clone": "^2.1.2", + "less": "^3.11.1", + "loader-utils": "^2.0.0", + "schema-utils": "^2.6.6" + }, + "dependencies": { + "clone": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "dev": true + }, + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true + }, + "loader-utils": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/loader-utils/-/loader-utils-2.0.2.tgz", + "integrity": "sha512-TM57VeHptv569d/GKh6TAYdzKblwDNiumOdkFnejjD0XwTH87K90w3O7AiJRqdQoXygvi1VQTJTLGhJl7WqA7A==", + "dev": true, + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + } + }, + "schema-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmmirror.com/schema-utils/-/schema-utils-2.7.1.tgz", + "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.5", + "ajv": "^6.12.4", + "ajv-keywords": "^3.5.2" + } + } + } + }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "libnpmaccess": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/libnpmaccess/-/libnpmaccess-4.0.3.tgz", + "integrity": "sha512-sPeTSNImksm8O2b6/pf3ikv4N567ERYEpeKRPSmqlNt1dTZbvgpJIzg5vAhXHpw2ISBsELFRelk0jEahj1c6nQ==", + "dev": true, + "requires": { + "aproba": "^2.0.0", + "minipass": "^3.1.1", + "npm-package-arg": "^8.1.2", + "npm-registry-fetch": "^11.0.0" + }, + "dependencies": { + "aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "dev": true + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "libnpmpublish": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/libnpmpublish/-/libnpmpublish-4.0.2.tgz", + "integrity": "sha512-+AD7A2zbVeGRCFI2aO//oUmapCwy7GHqPXFJh3qpToSRNU+tXKJ2YFUgjt04LPPAf2dlEH95s6EhIHM1J7bmOw==", + "dev": true, + "requires": { + "normalize-package-data": "^3.0.2", + "npm-package-arg": "^8.1.2", + "npm-registry-fetch": "^11.0.0", + "semver": "^7.1.3", + "ssri": "^8.0.1" + }, + "dependencies": { + "hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "normalize-package-data": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", + "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", + "dev": true, + "requires": { + "hosted-git-info": "^4.0.1", + "is-core-module": "^2.5.0", + "semver": "^7.3.4", + "validate-npm-package-license": "^3.0.1" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "dev": true, + "requires": { + "minipass": "^3.1.1" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true + }, + "lint-staged": { + "version": "8.2.1", + "resolved": "https://registry.npmmirror.com/lint-staged/-/lint-staged-8.2.1.tgz", + "integrity": "sha512-n0tDGR/rTCgQNwXnUf/eWIpPNddGWxC32ANTNYsj2k02iZb7Cz5ox2tytwBu+2r0zDXMEMKw7Y9OD/qsav561A==", + "dev": true, + "requires": { + "chalk": "^2.3.1", + "commander": "^2.14.1", + "cosmiconfig": "^5.2.0", + "debug": "^3.1.0", + "dedent": "^0.7.0", + "del": "^3.0.0", + "execa": "^1.0.0", + "g-status": "^2.0.2", + "is-glob": "^4.0.0", + "is-windows": "^1.0.2", + "listr": "^0.14.2", + "listr-update-renderer": "^0.5.0", + "lodash": "^4.17.11", + "log-symbols": "^2.2.0", + "micromatch": "^3.1.8", + "npm-which": "^3.0.1", + "p-map": "^1.1.1", + "path-is-inside": "^1.0.2", + "pify": "^3.0.0", + "please-upgrade-node": "^3.0.2", + "staged-git-files": "1.1.2", + "string-argv": "^0.0.2", + "stringify-object": "^3.2.2", + "yup": "^0.27.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "del": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/del/-/del-3.0.0.tgz", + "integrity": "sha512-7yjqSoVSlJzA4t/VUwazuEagGeANEKB3f/aNI//06pfKgwoCb7f6Q1gETN1sZzYaj6chTQ0AhIwDiPdfOjko4A==", + "dev": true, + "requires": { + "globby": "^6.1.0", + "is-path-cwd": "^1.0.0", + "is-path-in-cwd": "^1.0.0", + "p-map": "^1.1.1", + "pify": "^3.0.0", + "rimraf": "^2.2.8" + } + }, + "globby": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/globby/-/globby-6.1.0.tgz", + "integrity": "sha512-KVbFv2TQtbzCoxAnfD6JcHZTYCzyliEaaeM/gH8qQdkKr5s0OP9scEgvdcngyk7AVdY6YVW/TJHd+lQ/Df3Daw==", + "dev": true, + "requires": { + "array-union": "^1.0.1", + "glob": "^7.0.3", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true + } + } + }, + "is-path-cwd": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-path-cwd/-/is-path-cwd-1.0.0.tgz", + "integrity": "sha512-cnS56eR9SPAscL77ik76ATVqoPARTqPIVkMDVxRaWH06zT+6+CzIroYRJ0VVvm0Z1zfAvxvz9i/D3Ppjaqt5Nw==", + "dev": true + }, + "is-path-in-cwd": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz", + "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==", + "dev": true, + "requires": { + "is-path-inside": "^1.0.0" + } + }, + "is-path-inside": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-path-inside/-/is-path-inside-1.0.1.tgz", + "integrity": "sha512-qhsCR/Esx4U4hg/9I19OVUAJkGWtjRYHMRgUMZE2TDdj+Ag+kttZanLupfddNyglzz50cUlmWzUaI37GDfNx/g==", + "dev": true, + "requires": { + "path-is-inside": "^1.0.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "p-map": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-1.2.0.tgz", + "integrity": "sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA==", + "dev": true + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true + } + } + }, + "listr": { + "version": "0.14.3", + "resolved": "https://registry.npmmirror.com/listr/-/listr-0.14.3.tgz", + "integrity": "sha512-RmAl7su35BFd/xoMamRjpIE4j3v+L28o8CT5YhAXQJm1fD+1l9ngXY8JAQRJ+tFK2i5njvi0iRUKV09vPwA0iA==", + "dev": true, + "requires": { + "@samverschueren/stream-to-observable": "^0.3.0", + "is-observable": "^1.1.0", + "is-promise": "^2.1.0", + "is-stream": "^1.1.0", + "listr-silent-renderer": "^1.1.1", + "listr-update-renderer": "^0.5.0", + "listr-verbose-renderer": "^0.5.0", + "p-map": "^2.0.0", + "rxjs": "^6.3.3" + } + }, + "listr-silent-renderer": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz", + "integrity": "sha512-L26cIFm7/oZeSNVhWB6faeorXhMg4HNlb/dS/7jHhr708jxlXrtrBWo4YUxZQkc6dGoxEAe6J/D3juTRBUzjtA==", + "dev": true + }, + "listr-update-renderer": { + "version": "0.5.0", + "resolved": "https://registry.npmmirror.com/listr-update-renderer/-/listr-update-renderer-0.5.0.tgz", + "integrity": "sha512-tKRsZpKz8GSGqoI/+caPmfrypiaq+OQCbd+CovEC24uk1h952lVj5sC7SqyFUm+OaJ5HN/a1YLt5cit2FMNsFA==", + "dev": true, + "requires": { + "chalk": "^1.1.3", + "cli-truncate": "^0.2.1", + "elegant-spinner": "^1.0.1", + "figures": "^1.7.0", + "indent-string": "^3.0.0", + "log-symbols": "^1.0.2", + "log-update": "^2.3.0", + "strip-ansi": "^3.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "figures": { + "version": "1.7.0", + "resolved": "https://registry.npmmirror.com/figures/-/figures-1.7.0.tgz", + "integrity": "sha512-UxKlfCRuCBxSXU4C6t9scbDyWZ4VlaFFdojKtzJuSkuOBQ5CNFum+zZXFwHjo+CxBC1t6zlYPgHIgFjL8ggoEQ==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5", + "object-assign": "^4.1.0" + } + }, + "indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha512-BYqTHXTGUIvg7t1r4sJNKcbDZkL92nkXA8YtRpbjFHRHGDL/NtUeiBJMeE60kIFN/Mg8ESaWQvftaYMGJzQZCQ==", + "dev": true + }, + "log-symbols": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/log-symbols/-/log-symbols-1.0.2.tgz", + "integrity": "sha512-mmPrW0Fh2fxOzdBbFv4g1m6pR72haFLPJ2G5SJEELf1y+iaQrDG6cWCPjy54RHYbZAt7X+ls690Kw62AdWXBzQ==", + "dev": true, + "requires": { + "chalk": "^1.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true + } + } + }, + "listr-verbose-renderer": { + "version": "0.5.0", + "resolved": "https://registry.npmmirror.com/listr-verbose-renderer/-/listr-verbose-renderer-0.5.0.tgz", + "integrity": "sha512-04PDPqSlsqIOaaaGZ+41vq5FejI9auqTInicFRndCBgE3bXG8D6W1I+mWhk+1nqbHmyhla/6BUrd5OSiHwKRXw==", + "dev": true, + "requires": { + "chalk": "^2.4.1", + "cli-cursor": "^2.1.0", + "date-fns": "^1.27.2", + "figures": "^2.0.0" + } + }, + "load-json-file": { + "version": "6.2.0", + "resolved": "https://registry.npmmirror.com/load-json-file/-/load-json-file-6.2.0.tgz", + "integrity": "sha512-gUD/epcRms75Cw8RT1pUdHugZYM5ce64ucs2GEISABwkRsOQr0q2wm/MV2TKThycIe5e0ytRweW2RZxclogCdQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.15", + "parse-json": "^5.0.0", + "strip-bom": "^4.0.0", + "type-fest": "^0.6.0" + }, + "dependencies": { + "parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + } + }, + "strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true + } + } + }, + "loader-fs-cache": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/loader-fs-cache/-/loader-fs-cache-1.0.3.tgz", + "integrity": "sha512-ldcgZpjNJj71n+2Mf6yetz+c9bM4xpKtNds4LbqXzU/PTdeAX0g3ytnU1AJMEcTk2Lex4Smpe3Q/eCTsvUBxbA==", + "dev": true, + "requires": { + "find-cache-dir": "^0.1.1", + "mkdirp": "^0.5.1" + }, + "dependencies": { + "find-cache-dir": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/find-cache-dir/-/find-cache-dir-0.1.1.tgz", + "integrity": "sha512-Z9XSBoNE7xQiV6MSgPuCfyMokH2K7JdpRkOYE1+mu3d4BFJtx3GW+f6Bo4q8IX6rlf5MYbLBKW0pjl2cWdkm2A==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "mkdirp": "^0.5.1", + "pkg-dir": "^1.0.0" + } + }, + "find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha512-jvElSjyuo4EMQGoTwo1uJU5pQMwTW5lS1x05zzfJuTIyLR3zwO27LYrxNg+dlvKpGOuGy/MzBdXh80g0ve5+HA==", + "dev": true, + "requires": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha512-yTltuKuhtNeFJKa1PiRzfLAU5182q1y4Eb4XCJ3PBqyzEDkAZRzBrKKBct682ls9reBVHf9udYLN5Nd+K1B9BQ==", + "dev": true, + "requires": { + "pinkie-promise": "^2.0.0" + } + }, + "pkg-dir": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/pkg-dir/-/pkg-dir-1.0.0.tgz", + "integrity": "sha512-c6pv3OE78mcZ92ckebVDqg0aWSoKhOTbwCV6qbCWMk546mAL9pZln0+QsN/yQ7fkucd4+yJPLrCBXNt8Ruk+Eg==", + "dev": true, + "requires": { + "find-up": "^1.0.0" + } + } + } + }, + "loader-runner": { + "version": "2.4.0", + "resolved": "https://registry.npmmirror.com/loader-runner/-/loader-runner-2.4.0.tgz", + "integrity": "sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==" + }, + "loader-utils": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/loader-utils/-/loader-utils-1.4.0.tgz", + "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^1.0.1" + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmmirror.com/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "lodash._reinterpolate": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz", + "integrity": "sha512-xYHt68QRoYGjeeM/XOE1uJtvXQAgvszfBhjV4yvsQH0u2i9I6cI6c6/eG4Hh3UAOVn0y/xAXwmTzEay49Q//HA==", + "dev": true + }, + "lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmmirror.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "dev": true + }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "dev": true + }, + "lodash.defaultsdeep": { + "version": "4.6.1", + "resolved": "https://registry.npmmirror.com/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.1.tgz", + "integrity": "sha512-3j8wdDzYuWO3lM3Reg03MuQR957t287Rpcxp1njpEa8oDrikb+FwGdW3n+FELh/A6qib6yPit0j/pv9G/yeAqA==", + "dev": true + }, + "lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmmirror.com/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", + "dev": true + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmmirror.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", + "dev": true + }, + "lodash.ismatch": { + "version": "4.4.0", + "resolved": "https://registry.npmmirror.com/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", + "integrity": "sha512-fPMfXjGQEV9Xsq/8MTSgUf255gawYRbjwMyDbcvDhXgV7enSZA0hynz6vMPnpAb5iONEzBHBPsT+0zes5Z301g==", + "dev": true + }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmmirror.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "dev": true + }, + "lodash.kebabcase": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/lodash.kebabcase/-/lodash.kebabcase-4.1.1.tgz", + "integrity": "sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g==", + "dev": true + }, + "lodash.mapvalues": { + "version": "4.6.0", + "resolved": "https://registry.npmmirror.com/lodash.mapvalues/-/lodash.mapvalues-4.6.0.tgz", + "integrity": "sha512-JPFqXFeZQ7BfS00H58kClY7SPVeHertPE0lNuCyZ26/XlN8TvakYD7b9bGyNmXbT/D3BbtPAAmq90gPWqLkxlQ==", + "dev": true + }, + "lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", + "dev": true + }, + "lodash.template": { + "version": "4.5.0", + "resolved": "https://registry.npmmirror.com/lodash.template/-/lodash.template-4.5.0.tgz", + "integrity": "sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==", + "dev": true, + "requires": { + "lodash._reinterpolate": "^3.0.0", + "lodash.templatesettings": "^4.0.0" + } + }, + "lodash.templatesettings": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz", + "integrity": "sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==", + "dev": true, + "requires": { + "lodash._reinterpolate": "^3.0.0" + } + }, + "lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==" + }, + "lodash.transform": { + "version": "4.6.0", + "resolved": "https://registry.npmmirror.com/lodash.transform/-/lodash.transform-4.6.0.tgz", + "integrity": "sha512-LO37ZnhmBVx0GvOU/caQuipEh4GN82TcWv3yHlebGDgOxbxiwwzW5Pcx2AcvpIv2WmvmSMoC492yQFNhy/l/UQ==", + "dev": true + }, + "lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmmirror.com/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", + "dev": true + }, + "lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmmirror.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "dev": true + }, + "log-symbols": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/log-symbols/-/log-symbols-2.2.0.tgz", + "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==", + "dev": true, + "requires": { + "chalk": "^2.0.1" + } + }, + "log-update": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/log-update/-/log-update-2.3.0.tgz", + "integrity": "sha512-vlP11XfFGyeNQlmEn9tJ66rEW1coA/79m5z6BCkudjbAGE83uhAcGYrBFwfs3AdLiLzGRusRPAbSPK9xZteCmg==", + "dev": true, + "requires": { + "ansi-escapes": "^3.0.0", + "cli-cursor": "^2.0.0", + "wrap-ansi": "^3.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", + "dev": true + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "wrap-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz", + "integrity": "sha512-iXR3tDXpbnTpzjKSylUJRkLuOrEC7hwEB221cgn6wtF8wpmz28puFXAEfPT5zrjM3wahygB//VuWEr1vTkDcNQ==", + "dev": true, + "requires": { + "string-width": "^2.1.1", + "strip-ansi": "^4.0.0" + } + } + } + }, + "loglevel": { + "version": "1.8.0", + "resolved": "https://registry.npmmirror.com/loglevel/-/loglevel-1.8.0.tgz", + "integrity": "sha512-G6A/nJLRgWOuuwdNuA6koovfEV1YpqqAG4pRUlFaz3jj2QNZ8M4vBqnVA+HBTmU/AMNUtlOsMmSpF6NyOjztbA==", + "dev": true + }, + "loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dev": true, + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + }, + "loud-rejection": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/loud-rejection/-/loud-rejection-1.6.0.tgz", + "integrity": "sha512-RPNliZOFkqFumDhvYqOaNY4Uz9oJM2K9tC6JWsJJsNdhuONW4LQHRBpb0qf4pJApVffI5N39SwzWZJuEhfd7eQ==", + "dev": true, + "requires": { + "currently-unhandled": "^0.4.1", + "signal-exit": "^3.0.0" + } + }, + "lower-case": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/lower-case/-/lower-case-1.1.4.tgz", + "integrity": "sha512-2Fgx1Ycm599x+WGpIYwJOvsjmXFzTSc34IwDWALRA/8AopUKAVPwfJ+h5+f85BCp0PWmmJcWzEpxOpoXycMpdA==", + "dev": true + }, + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "requires": { + "yallist": "^3.0.2" + } + }, + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + } + }, + "make-fetch-happen": { + "version": "9.1.0", + "resolved": "https://registry.npmmirror.com/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz", + "integrity": "sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==", + "dev": true, + "requires": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.2.0", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.2", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^6.0.0", + "ssri": "^8.0.0" + }, + "dependencies": { + "cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmmirror.com/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "dev": true, + "requires": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + } + }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "dev": true, + "requires": { + "minipass": "^3.1.1" + } + }, + "tar": { + "version": "6.1.11", + "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmmirror.com/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==" + }, + "map-obj": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/map-obj/-/map-obj-4.3.0.tgz", + "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", + "dev": true + }, + "map-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/map-visit/-/map-visit-1.0.0.tgz", + "integrity": "sha512-4y7uGv8bd2WdM9vpQsiQNo41Ln1NvhvDRuVt0k2JZQ+ezN2uaQes7lZeZ+QQUHOLQAtDaBJ+7wCbi+ab/KFs+w==", + "requires": { + "object-visit": "^1.0.0" + } + }, + "matcher": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/matcher/-/matcher-1.1.1.tgz", + "integrity": "sha512-+BmqxWIubKTRKNWx/ahnCkk3mG8m7OturVlqq6HiojGJTd5hVYbgZm6WzcYPCoB+KBT4Vd6R7WSRG2OADNaCjg==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.4" + } + }, + "math-random": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/math-random/-/math-random-1.0.4.tgz", + "integrity": "sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A==", + "dev": true + }, + "md5": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "requires": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, + "md5.js": { + "version": "1.3.5", + "resolved": "https://registry.npmmirror.com/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "requires": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "mdn-data": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/mdn-data/-/mdn-data-2.0.4.tgz", + "integrity": "sha512-iV3XNKw06j5Q7mi6h+9vbx23Tv7JkjEVgKHW4pimwyDGWm0OIQntJJ+u1C6mg6mK1EaTv42XQ7w76yuzH7M2cA==" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true + }, + "memory-fs": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/memory-fs/-/memory-fs-0.4.1.tgz", + "integrity": "sha512-cda4JKCxReDXFXRqOHPQscuIYg1PvxbE2S2GP45rnwfEK+vZaXC8C1OFvdHIbgw0DLzowXGVoxLaAmlgRy14GQ==", + "requires": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + } + }, + "meow": { + "version": "8.1.2", + "resolved": "https://registry.npmmirror.com/meow/-/meow-8.1.2.tgz", + "integrity": "sha512-r85E3NdZ+mpYk1C6RjPFEMSE+s1iZMuHtsHAqY0DT3jZczl0diWUZ8g6oU7h0M9cD2EL+PzaYghhCLzR0ZNn5Q==", + "dev": true, + "requires": { + "@types/minimist": "^1.2.0", + "camelcase-keys": "^6.2.2", + "decamelize-keys": "^1.1.0", + "hard-rejection": "^2.1.0", + "minimist-options": "4.1.0", + "normalize-package-data": "^3.0.0", + "read-pkg-up": "^7.0.1", + "redent": "^3.0.0", + "trim-newlines": "^3.0.0", + "type-fest": "^0.18.0", + "yargs-parser": "^20.2.3" + }, + "dependencies": { + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "normalize-package-data": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", + "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", + "dev": true, + "requires": { + "hosted-git-info": "^4.0.1", + "is-core-module": "^2.5.0", + "semver": "^7.3.4", + "validate-npm-package-license": "^3.0.1" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "read-pkg-up": { + "version": "7.0.1", + "resolved": "https://registry.npmmirror.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz", + "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==", + "dev": true, + "requires": { + "find-up": "^4.1.0", + "read-pkg": "^5.2.0", + "type-fest": "^0.8.1" + }, + "dependencies": { + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true + } + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "type-fest": { + "version": "0.18.1", + "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.18.1.tgz", + "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==", + "dev": true + }, + "merge-options": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/merge-options/-/merge-options-1.0.1.tgz", + "integrity": "sha512-iuPV41VWKWBIOpBsjoxjDZw8/GbSfZ2mk7N1453bwMrfzdrIk7EzBd+8UVR6rkw67th7xnk9Dytl3J+lHPdxvg==", + "dev": true, + "requires": { + "is-plain-obj": "^1.1" + } + }, + "merge-source-map": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/merge-source-map/-/merge-source-map-1.1.0.tgz", + "integrity": "sha512-Qkcp7P2ygktpMPh2mCQZaf3jhN6D3Z/qVZHSdWvQ+2Ef5HgRAPBO57A77+ENm0CPx2+1Ce/MYKi3ymqdfuqibw==", + "dev": true, + "requires": { + "source-map": "^0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true + }, + "micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmmirror.com/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + } + }, + "miller-rabin": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/miller-rabin/-/miller-rabin-4.0.1.tgz", + "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", + "requires": { + "bn.js": "^4.0.0", + "brorand": "^1.0.1" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmmirror.com/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "mime": { + "version": "2.6.0", + "resolved": "https://registry.npmmirror.com/mime/-/mime-2.6.0.tgz", + "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==", + "dev": true + }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmmirror.com/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmmirror.com/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "requires": { + "mime-db": "1.52.0" + } + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, + "min-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/min-indent/-/min-indent-1.0.1.tgz", + "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", + "dev": true + }, + "mini-css-extract-plugin": { + "version": "0.8.2", + "resolved": "https://registry.npmmirror.com/mini-css-extract-plugin/-/mini-css-extract-plugin-0.8.2.tgz", + "integrity": "sha512-a3Y4of27Wz+mqK3qrcd3VhYz6cU0iW5x3Sgvqzbj+XmlrSizmvu8QQMl5oMYJjgHOC4iyt+w7l4umP+dQeW3bw==", + "dev": true, + "requires": { + "loader-utils": "^1.1.0", + "normalize-url": "1.9.1", + "schema-utils": "^1.0.0", + "webpack-sources": "^1.1.0" + }, + "dependencies": { + "normalize-url": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/normalize-url/-/normalize-url-1.9.1.tgz", + "integrity": "sha512-A48My/mtCklowHBlI8Fq2jFWK4tX4lJ5E6ytFsSOq1fzpvT0SQSgKhSg7lN5c2uYFOrUAOQp6zhhJnpp1eMloQ==", + "dev": true, + "requires": { + "object-assign": "^4.0.1", + "prepend-http": "^1.0.0", + "query-string": "^4.1.0", + "sort-keys": "^1.0.0" + } + } + } + }, + "minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==" + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmmirror.com/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + }, + "minimist-options": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/minimist-options/-/minimist-options-4.1.0.tgz", + "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", + "dev": true, + "requires": { + "arrify": "^1.0.1", + "is-plain-obj": "^1.1.0", + "kind-of": "^6.0.3" + }, + "dependencies": { + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", + "dev": true + } + } + }, + "minipass": { + "version": "2.9.0", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", + "dev": true, + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + } + }, + "minipass-collect": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/minipass-collect/-/minipass-collect-1.0.2.tgz", + "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "minipass-fetch": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/minipass-fetch/-/minipass-fetch-1.4.1.tgz", + "integrity": "sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==", + "dev": true, + "requires": { + "encoding": "^0.1.12", + "minipass": "^3.1.0", + "minipass-sized": "^1.0.3", + "minizlib": "^2.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "minipass-json-stream": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/minipass-json-stream/-/minipass-json-stream-1.0.1.tgz", + "integrity": "sha512-ODqY18UZt/I8k+b7rl2AENgbWE8IDYam+undIJONvigAz8KR5GWblsFTEfQs0WODsjbSXWlm+JHEv8Gr6Tfdbg==", + "dev": true, + "requires": { + "jsonparse": "^1.3.1", + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "minizlib": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", + "dev": true, + "requires": { + "minipass": "^2.9.0" + } + }, + "mississippi": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/mississippi/-/mississippi-3.0.0.tgz", + "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==", + "requires": { + "concat-stream": "^1.5.0", + "duplexify": "^3.4.2", + "end-of-stream": "^1.1.0", + "flush-write-stream": "^1.0.0", + "from2": "^2.1.0", + "parallel-transform": "^1.1.0", + "pump": "^3.0.0", + "pumpify": "^1.3.3", + "stream-each": "^1.1.0", + "through2": "^2.0.0" + } + }, + "mitt": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/mitt/-/mitt-1.1.2.tgz", + "integrity": "sha512-3btxP0O9iGADGWAkteQ8mzDtEspZqu4I32y4GZYCV5BrwtzdcRpF4dQgNdJadCrbBx7Lu6Sq9AVrerMHR0Hkmw==", + "dev": true + }, + "mixin-deep": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "requires": { + "for-in": "^1.0.2", + "is-extendable": "^1.0.1" + }, + "dependencies": { + "is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "requires": { + "is-plain-object": "^2.0.4" + } + } + } + }, + "mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "requires": { + "minimist": "^1.2.6" + } + }, + "mkdirp-infer-owner": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/mkdirp-infer-owner/-/mkdirp-infer-owner-2.0.0.tgz", + "integrity": "sha512-sdqtiFt3lkOaYvTXSRIUjkIdPTcxgv5+fgqYE/5qgwdw12cOrAuzzgzvVExIkH/ul1oeHN3bCLOWSG3XOqbKKw==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "infer-owner": "^1.0.4", + "mkdirp": "^1.0.3" + }, + "dependencies": { + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + } + } + }, + "ml-array-max": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/ml-array-max/-/ml-array-max-1.2.4.tgz", + "integrity": "sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==", + "requires": { + "is-any-array": "^2.0.0" + } + }, + "ml-array-min": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/ml-array-min/-/ml-array-min-1.2.3.tgz", + "integrity": "sha512-VcZ5f3VZ1iihtrGvgfh/q0XlMobG6GQ8FsNyQXD3T+IlstDv85g8kfV0xUG1QPRO/t21aukaJowDzMTc7j5V6Q==", + "requires": { + "is-any-array": "^2.0.0" + } + }, + "ml-array-rescale": { + "version": "1.3.7", + "resolved": "https://registry.npmmirror.com/ml-array-rescale/-/ml-array-rescale-1.3.7.tgz", + "integrity": "sha512-48NGChTouvEo9KBctDfHC3udWnQKNKEWN0ziELvY3KG25GR5cA8K8wNVzracsqSW1QEkAXjTNx+ycgAv06/1mQ==", + "requires": { + "is-any-array": "^2.0.0", + "ml-array-max": "^1.2.4", + "ml-array-min": "^1.2.3" + } + }, + "ml-matrix": { + "version": "6.10.0", + "resolved": "https://registry.npmmirror.com/ml-matrix/-/ml-matrix-6.10.0.tgz", + "integrity": "sha512-wU+jacx1dcP1QArV1/Kv49Ah6y2fq+BiQl2BnNVBC+hoCW7KgBZ4YZrowPopeoY164TB6Kes5wMeDjY8ODHYDg==", + "requires": { + "is-any-array": "^2.0.0", + "ml-array-rescale": "^1.3.7" + } + }, + "mockjs": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/mockjs/-/mockjs-1.1.0.tgz", + "integrity": "sha512-eQsKcWzIaZzEZ07NuEyO4Nw65g0hdWAyurVol1IPl1gahRwY+svqzfgfey8U8dahLwG44d6/RwEzuK52rSa/JQ==", + "dev": true, + "requires": { + "commander": "*" + } + }, + "modify-values": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/modify-values/-/modify-values-1.0.1.tgz", + "integrity": "sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==", + "dev": true + }, + "moment": { + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==" + }, + "monaco-editor": { + "version": "0.19.3", + "resolved": "https://registry.npmmirror.com/monaco-editor/-/monaco-editor-0.19.3.tgz", + "integrity": "sha512-2n1vJBVQF2Hhi7+r1mMeYsmlf18hjVb6E0v5SoMZyb4aeOmYPKun+CE3gYpiNA1KEvtSdaDHFBqH9d7Wd9vREg==" + }, + "monaco-editor-webpack-plugin": { + "version": "1.8.2", + "resolved": "https://registry.npmmirror.com/monaco-editor-webpack-plugin/-/monaco-editor-webpack-plugin-1.8.2.tgz", + "integrity": "sha512-g9G7A/lxQtpPsYaZFBqm73dwVkOziGUXExIR6iW7ksZUaiMkpvdTiE9O8edgdJGo+XtCmjycmIKB1Lt8VKbSTQ==", + "dev": true, + "requires": { + "loader-utils": "^1.2.3" + } + }, + "move-concurrently": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/move-concurrently/-/move-concurrently-1.0.1.tgz", + "integrity": "sha512-hdrFxZOycD/g6A6SoI2bB5NA/5NEqD0569+S47WZhPvm46sD50ZHdYaFmnua5lndde9rCHGjmfK7Z8BuCt/PcQ==", + "requires": { + "aproba": "^1.1.1", + "copy-concurrently": "^1.0.0", + "fs-write-stream-atomic": "^1.0.8", + "mkdirp": "^0.5.1", + "rimraf": "^2.5.4", + "run-queue": "^1.0.3" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "multicast-dns": { + "version": "6.2.3", + "resolved": "https://registry.npmmirror.com/multicast-dns/-/multicast-dns-6.2.3.tgz", + "integrity": "sha512-ji6J5enbMyGRHIAkAOu3WdV8nggqviKCEKtXcOqfphZZtQrmHKycfynJ2V7eVPUA4NhJ6V7Wf4TmGbTwKE9B6g==", + "dev": true, + "requires": { + "dns-packet": "^1.3.1", + "thunky": "^1.0.2" + } + }, + "multicast-dns-service-types": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz", + "integrity": "sha512-cnAsSVxIDsYt0v7HmC0hWZFwwXSh+E6PgCrREDuN/EsjgLwA5XRmlMHhSiDPrt6HxY1gTivEa/Zh7GtODoLevQ==", + "dev": true + }, + "multimatch": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/multimatch/-/multimatch-5.0.0.tgz", + "integrity": "sha512-ypMKuglUrZUD99Tk2bUQ+xNQj43lPEfAeX2o9cTteAmShXy2VHDJpuwu1o0xqoKCt9jLVAvwyFKdLTPXKAfJyA==", + "dev": true, + "requires": { + "@types/minimatch": "^3.0.3", + "array-differ": "^3.0.0", + "array-union": "^2.1.0", + "arrify": "^2.0.1", + "minimatch": "^3.0.4" + }, + "dependencies": { + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true + } + } + }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmmirror.com/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==", + "dev": true + }, + "mv": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/mv/-/mv-2.1.1.tgz", + "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==", + "dev": true, + "requires": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + }, + "dependencies": { + "glob": { + "version": "6.0.4", + "resolved": "https://registry.npmmirror.com/glob/-/glob-6.0.4.tgz", + "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", + "dev": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.4.5", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", + "dev": true, + "requires": { + "glob": "^6.0.1" + } + } + } + }, + "mz": { + "version": "2.7.0", + "resolved": "https://registry.npmmirror.com/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "requires": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "nan": { + "version": "2.15.0", + "resolved": "https://registry.npmmirror.com/nan/-/nan-2.15.0.tgz", + "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==" + }, + "nanomatch": { + "version": "1.2.13", + "resolved": "https://registry.npmmirror.com/nanomatch/-/nanomatch-1.2.13.tgz", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "fragment-cache": "^0.2.1", + "is-windows": "^1.0.2", + "kind-of": "^6.0.2", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + } + }, + "native-request": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/native-request/-/native-request-1.1.0.tgz", + "integrity": "sha512-uZ5rQaeRn15XmpgE0xoPL8YWqcX90VtCFglYwAgkvKM5e8fog+vePLAhHxuuv/gRkrQxIeh5U3q9sMNUrENqWw==", + "dev": true, + "optional": true + }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, + "ncp": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", + "dev": true + }, + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmmirror.com/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true + }, + "neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmmirror.com/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" + }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, + "no-case": { + "version": "2.3.2", + "resolved": "https://registry.npmmirror.com/no-case/-/no-case-2.3.2.tgz", + "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", + "dev": true, + "requires": { + "lower-case": "^1.1.1" + } + }, + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmmirror.com/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dev": true, + "requires": { + "whatwg-url": "^5.0.0" + }, + "dependencies": { + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmmirror.com/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } + }, + "node-forge": { + "version": "0.10.0", + "resolved": "https://registry.npmmirror.com/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==", + "dev": true + }, + "node-gyp": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/node-gyp/-/node-gyp-5.1.1.tgz", + "integrity": "sha512-WH0WKGi+a4i4DUt2mHnvocex/xPLp9pYt5R6M2JdFB7pJ7Z34hveZ4nDTGTiLXCkitA9T8HFZjhinBCiVHYcWw==", + "dev": true, + "requires": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.2", + "mkdirp": "^0.5.1", + "nopt": "^4.0.1", + "npmlog": "^4.1.2", + "request": "^2.88.0", + "rimraf": "^2.6.3", + "semver": "^5.7.1", + "tar": "^4.4.12", + "which": "^1.3.1" + } + }, + "node-ipc": { + "version": "9.2.1", + "resolved": "https://registry.npmmirror.com/node-ipc/-/node-ipc-9.2.1.tgz", + "integrity": "sha512-mJzaM6O3xHf9VT8BULvJSbdVbmHUKRNOH7zDDkCrA1/T+CVjq2WVIDfLt0azZRXpgArJtl3rtmEozrbXPZ9GaQ==", + "dev": true, + "requires": { + "event-pubsub": "4.3.0", + "js-message": "1.0.7", + "js-queue": "2.0.2" + } + }, + "node-libs-browser": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/node-libs-browser/-/node-libs-browser-2.2.1.tgz", + "integrity": "sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==", + "requires": { + "assert": "^1.1.1", + "browserify-zlib": "^0.2.0", + "buffer": "^4.3.0", + "console-browserify": "^1.1.0", + "constants-browserify": "^1.0.0", + "crypto-browserify": "^3.11.0", + "domain-browser": "^1.1.1", + "events": "^3.0.0", + "https-browserify": "^1.0.0", + "os-browserify": "^0.3.0", + "path-browserify": "0.0.1", + "process": "^0.11.10", + "punycode": "^1.2.4", + "querystring-es3": "^0.2.0", + "readable-stream": "^2.3.3", + "stream-browserify": "^2.0.1", + "stream-http": "^2.7.2", + "string_decoder": "^1.0.0", + "timers-browserify": "^2.0.4", + "tty-browserify": "0.0.0", + "url": "^0.11.0", + "util": "^0.11.0", + "vm-browserify": "^1.0.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==" + } + } + }, + "node-releases": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.3.tgz", + "integrity": "sha512-maHFz6OLqYxz+VQyCAtA3PTX4UP/53pa05fyDNc9CwjvJ0yEh6+xBwKsgCxMNhS8taUKBFYxfuiaD9U/55iFaw==", + "dev": true + }, + "node-sass": { + "version": "4.14.1", + "resolved": "https://registry.npmmirror.com/node-sass/-/node-sass-4.14.1.tgz", + "integrity": "sha512-sjCuOlvGyCJS40R8BscF5vhVlQjNN069NtQ1gSxyK1u9iqvn6tf7O1R4GNowVZfiZUCRt5MmMs1xd+4V/7Yr0g==", + "dev": true, + "requires": { + "async-foreach": "^0.1.3", + "chalk": "^1.1.1", + "cross-spawn": "^3.0.0", + "gaze": "^1.0.0", + "get-stdin": "^4.0.1", + "glob": "^7.0.3", + "in-publish": "^2.0.0", + "lodash": "^4.17.15", + "meow": "^3.7.0", + "mkdirp": "^0.5.1", + "nan": "^2.13.2", + "node-gyp": "^3.8.0", + "npmlog": "^4.0.0", + "request": "^2.88.0", + "sass-graph": "2.2.5", + "stdout-stream": "^1.4.0", + "true-case-path": "^1.0.2" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true + }, + "camelcase": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/camelcase/-/camelcase-2.1.1.tgz", + "integrity": "sha512-DLIsRzJVBQu72meAKPkWQOLcujdXT32hwdfnkI1frSiSRMK1MofjKHf+MEx0SB6fjEFXL8fBDv1dKymBlOp4Qw==", + "dev": true + }, + "camelcase-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "integrity": "sha512-bA/Z/DERHKqoEOrp+qeGKw1QlvEQkGZSc0XaY6VnTxZr+Kv1G5zFwttpjv8qxZ/sBPT4nthwZaAcsAZTJlSKXQ==", + "dev": true, + "requires": { + "camelcase": "^2.0.0", + "map-obj": "^1.0.0" + } + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "cross-spawn": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-3.0.1.tgz", + "integrity": "sha512-eZ+m1WNhSZutOa/uRblAc9Ut5MQfukFrFMtPSm3bZCA888NmMd5AWXWdgRZ80zd+pTk1P2JrGjg9pUPTvl2PWQ==", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + }, + "find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha512-jvElSjyuo4EMQGoTwo1uJU5pQMwTW5lS1x05zzfJuTIyLR3zwO27LYrxNg+dlvKpGOuGy/MzBdXh80g0ve5+HA==", + "dev": true, + "requires": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "get-stdin": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/get-stdin/-/get-stdin-4.0.1.tgz", + "integrity": "sha512-F5aQMywwJ2n85s4hJPTT9RPxGmubonuB10MNYo17/xph174n2MIR33HRguhzVag10O/npM7SPk73LMZNP+FaWw==", + "dev": true + }, + "indent-string": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/indent-string/-/indent-string-2.1.0.tgz", + "integrity": "sha512-aqwDFWSgSgfRaEwao5lg5KEcVd/2a+D1rvoG7NdilmYz0NwRk6StWpWdz/Hpk34MKPpx7s8XxUqimfcQK6gGlg==", + "dev": true, + "requires": { + "repeating": "^2.0.0" + } + }, + "load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha512-cy7ZdNRXdablkXYNI049pthVeXFurRyb9+hA/dZzerZ0pGTx42z+y+ssxBaVV2l70t1muq5IdKhn4UtcoGUY9A==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0", + "strip-bom": "^2.0.0" + } + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", + "dev": true + }, + "meow": { + "version": "3.7.0", + "resolved": "https://registry.npmmirror.com/meow/-/meow-3.7.0.tgz", + "integrity": "sha512-TNdwZs0skRlpPpCUK25StC4VH+tP5GgeY1HQOOGP+lQ2xtdkN2VtT/5tiX9k3IWpkBPV9b3LsAWXn4GGi/PrSA==", + "dev": true, + "requires": { + "camelcase-keys": "^2.0.0", + "decamelize": "^1.1.2", + "loud-rejection": "^1.0.0", + "map-obj": "^1.0.1", + "minimist": "^1.1.3", + "normalize-package-data": "^2.3.4", + "object-assign": "^4.0.1", + "read-pkg-up": "^1.0.1", + "redent": "^1.0.0", + "trim-newlines": "^1.0.0" + } + }, + "node-gyp": { + "version": "3.8.0", + "resolved": "https://registry.npmmirror.com/node-gyp/-/node-gyp-3.8.0.tgz", + "integrity": "sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA==", + "dev": true, + "requires": { + "fstream": "^1.0.0", + "glob": "^7.0.3", + "graceful-fs": "^4.1.2", + "mkdirp": "^0.5.0", + "nopt": "2 || 3", + "npmlog": "0 || 1 || 2 || 3 || 4", + "osenv": "0", + "request": "^2.87.0", + "rimraf": "2", + "semver": "~5.3.0", + "tar": "^2.0.0", + "which": "1" + } + }, + "nopt": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/nopt/-/nopt-3.0.6.tgz", + "integrity": "sha512-4GUt3kSEYmk4ITxzB/b9vaIDfUVWN/Ml1Fwl11IlnIG2iaJ9O6WXZ9SrYM9NLI8OCBieN2Y8SWC2oJV0RQ7qYg==", + "dev": true, + "requires": { + "abbrev": "1" + } + }, + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==", + "dev": true, + "requires": { + "error-ex": "^1.2.0" + } + }, + "path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha512-yTltuKuhtNeFJKa1PiRzfLAU5182q1y4Eb4XCJ3PBqyzEDkAZRzBrKKBct682ls9reBVHf9udYLN5Nd+K1B9BQ==", + "dev": true, + "requires": { + "pinkie-promise": "^2.0.0" + } + }, + "path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha512-S4eENJz1pkiQn9Znv33Q+deTOKmbl+jj1Fl+qiP/vYezj+S8x+J3Uo0ISrx/QoEvIlOaDWJhPaRd1flJ9HXZqg==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true + }, + "read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha512-7BGwRHqt4s/uVbuyoeejRn4YmFnYZiFl4AuaeXHlgZf3sONF0SOGlxs2Pw8g6hCKupo08RafIO5YXFNOKTfwsQ==", + "dev": true, + "requires": { + "load-json-file": "^1.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^1.0.0" + } + }, + "read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha512-WD9MTlNtI55IwYUS27iHh9tK3YoIVhxis8yKhLpTqWtml739uXc9NWTpxoHkfZf3+DkCCsXox94/VWZniuZm6A==", + "dev": true, + "requires": { + "find-up": "^1.0.0", + "read-pkg": "^1.0.0" + } + }, + "redent": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/redent/-/redent-1.0.0.tgz", + "integrity": "sha512-qtW5hKzGQZqKoh6JNSD+4lfitfPKGz42e6QwiRmPM5mmKtR0N41AbJRYu0xJi7nhOJ4WDgRkKvAk6tw4WIwR4g==", + "dev": true, + "requires": { + "indent-string": "^2.1.0", + "strip-indent": "^1.0.1" + } + }, + "semver": { + "version": "5.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-5.3.0.tgz", + "integrity": "sha512-mfmm3/H9+67MCVix1h+IXTpDwL6710LyHuk7+cWC9T1mE0qz4iHhh6r4hU2wrIT9iTsAAC2XQRvfblL028cpLw==", + "dev": true + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==", + "dev": true, + "requires": { + "is-utf8": "^0.2.0" + } + }, + "strip-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/strip-indent/-/strip-indent-1.0.1.tgz", + "integrity": "sha512-I5iQq6aFMM62fBEAIB/hXzwJD6EEZ0xEGCX2t7oXqaKPIRgt4WruAQ285BISgdkP+HLGWyeGmNJcpIwFeRYRUA==", + "dev": true, + "requires": { + "get-stdin": "^4.0.1" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true + }, + "tar": { + "version": "2.2.2", + "resolved": "https://registry.npmmirror.com/tar/-/tar-2.2.2.tgz", + "integrity": "sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA==", + "dev": true, + "requires": { + "block-stream": "*", + "fstream": "^1.0.12", + "inherits": "2" + } + }, + "trim-newlines": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/trim-newlines/-/trim-newlines-1.0.0.tgz", + "integrity": "sha512-Nm4cF79FhSTzrLKGDMi3I4utBtFv8qKy4sq1enftf2gMdpqI8oVQTAfySkTz5r49giVzDj88SVZXP4CeYQwjaw==", + "dev": true + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", + "dev": true + } + } + }, + "nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "dev": true, + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, + "normalize-range": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/normalize-range/-/normalize-range-0.1.2.tgz", + "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", + "dev": true + }, + "normalize-url": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/normalize-url/-/normalize-url-3.3.0.tgz", + "integrity": "sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==", + "dev": true + }, + "npm-bundled": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/npm-bundled/-/npm-bundled-1.1.2.tgz", + "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==", + "dev": true, + "requires": { + "npm-normalize-package-bin": "^1.0.1" + } + }, + "npm-install-checks": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/npm-install-checks/-/npm-install-checks-4.0.0.tgz", + "integrity": "sha512-09OmyDkNLYwqKPOnbI8exiOZU2GVVmQp7tgez2BPi5OZC8M82elDAps7sxC4l//uSUtotWqoEIDwjRvWH4qz8w==", + "dev": true, + "requires": { + "semver": "^7.1.1" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "npm-lifecycle": { + "version": "3.1.5", + "resolved": "https://registry.npmmirror.com/npm-lifecycle/-/npm-lifecycle-3.1.5.tgz", + "integrity": "sha512-lDLVkjfZmvmfvpvBzA4vzee9cn+Me4orq0QF8glbswJVEbIcSNWib7qGOffolysc3teCqbbPZZkzbr3GQZTL1g==", + "dev": true, + "requires": { + "byline": "^5.0.0", + "graceful-fs": "^4.1.15", + "node-gyp": "^5.0.2", + "resolve-from": "^4.0.0", + "slide": "^1.1.6", + "uid-number": "0.0.6", + "umask": "^1.1.0", + "which": "^1.3.1" + }, + "dependencies": { + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + } + } + }, + "npm-normalize-package-bin": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", + "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==", + "dev": true + }, + "npm-package-arg": { + "version": "8.1.5", + "resolved": "https://registry.npmmirror.com/npm-package-arg/-/npm-package-arg-8.1.5.tgz", + "integrity": "sha512-LhgZrg0n0VgvzVdSm1oiZworPbTxYHUJCgtsJW8mGvlDpxTM1vSJc3m5QZeUkhAHIzbz3VCHd/R4osi1L1Tg/Q==", + "dev": true, + "requires": { + "hosted-git-info": "^4.0.1", + "semver": "^7.3.4", + "validate-npm-package-name": "^3.0.0" + }, + "dependencies": { + "hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "npm-packlist": { + "version": "2.2.2", + "resolved": "https://registry.npmmirror.com/npm-packlist/-/npm-packlist-2.2.2.tgz", + "integrity": "sha512-Jt01acDvJRhJGthnUJVF/w6gumWOZxO7IkpY/lsX9//zqQgnF7OJaxgQXcerd4uQOLu7W5bkb4mChL9mdfm+Zg==", + "dev": true, + "requires": { + "glob": "^7.1.6", + "ignore-walk": "^3.0.3", + "npm-bundled": "^1.1.1", + "npm-normalize-package-bin": "^1.0.1" + } + }, + "npm-path": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/npm-path/-/npm-path-2.0.4.tgz", + "integrity": "sha512-IFsj0R9C7ZdR5cP+ET342q77uSRdtWOlWpih5eC+lu29tIDbNEgDbzgVJ5UFvYHWhxDZ5TFkJafFioO0pPQjCw==", + "dev": true, + "requires": { + "which": "^1.2.10" + } + }, + "npm-pick-manifest": { + "version": "6.1.1", + "resolved": "https://registry.npmmirror.com/npm-pick-manifest/-/npm-pick-manifest-6.1.1.tgz", + "integrity": "sha512-dBsdBtORT84S8V8UTad1WlUyKIY9iMsAmqxHbLdeEeBNMLQDlDWWra3wYUx9EBEIiG/YwAy0XyNHDd2goAsfuA==", + "dev": true, + "requires": { + "npm-install-checks": "^4.0.0", + "npm-normalize-package-bin": "^1.0.1", + "npm-package-arg": "^8.1.2", + "semver": "^7.3.4" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "npm-registry-fetch": { + "version": "11.0.0", + "resolved": "https://registry.npmmirror.com/npm-registry-fetch/-/npm-registry-fetch-11.0.0.tgz", + "integrity": "sha512-jmlgSxoDNuhAtxUIG6pVwwtz840i994dL14FoNVZisrmZW5kWd63IUTNv1m/hyRSGSqWjCUp/YZlS1BJyNp9XA==", + "dev": true, + "requires": { + "make-fetch-happen": "^9.0.1", + "minipass": "^3.1.3", + "minipass-fetch": "^1.3.0", + "minipass-json-stream": "^1.0.1", + "minizlib": "^2.0.0", + "npm-package-arg": "^8.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==", + "dev": true, + "requires": { + "path-key": "^2.0.0" + } + }, + "npm-which": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/npm-which/-/npm-which-3.0.1.tgz", + "integrity": "sha512-CM8vMpeFQ7MAPin0U3wzDhSGV0hMHNwHU0wjo402IVizPDrs45jSfSuoC+wThevY88LQti8VvaAnqYAeVy3I1A==", + "dev": true, + "requires": { + "commander": "^2.9.0", + "npm-path": "^2.0.2", + "which": "^1.2.10" + } + }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "dev": true, + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "requires": { + "boolbase": "~1.0.0" + } + }, + "num2fraction": { + "version": "1.2.2", + "resolved": "https://registry.npmmirror.com/num2fraction/-/num2fraction-1.2.2.tgz", + "integrity": "sha512-Y1wZESM7VUThYY+4W+X4ySH2maqcA+p7UR+w8VWNWVAd6lwuXXWz/w/Cz43J/dI2I+PS6wD5N+bJUF+gjWvIqg==", + "dev": true + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==", + "dev": true + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmmirror.com/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, + "object-copy": { + "version": "0.1.0", + "resolved": "https://registry.npmmirror.com/object-copy/-/object-copy-0.1.0.tgz", + "integrity": "sha512-79LYn6VAb63zgtmAteVOWo9Vdj71ZVBy3Pbse+VqxDpEP83XuujMrGqHIwAXJ5I/aM0zU7dIyIAhifVTPrNItQ==", + "requires": { + "copy-descriptor": "^0.1.0", + "define-property": "^0.2.5", + "kind-of": "^3.0.3" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "object-hash": { + "version": "1.3.1", + "resolved": "https://registry.npmmirror.com/object-hash/-/object-hash-1.3.1.tgz", + "integrity": "sha512-OSuu/pU4ENM9kmREg0BdNrUDIl1heYa4mBZacJc+vVWz4GtAwu7jO8s4AIt2aGRUTqxykpWzI3Oqnsm13tTMDA==", + "dev": true + }, + "object-inspect": { + "version": "1.12.0", + "resolved": "https://registry.npmmirror.com/object-inspect/-/object-inspect-1.12.0.tgz", + "integrity": "sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g==" + }, + "object-is": { + "version": "1.1.5", + "resolved": "https://registry.npmmirror.com/object-is/-/object-is-1.1.5.tgz", + "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, + "object-visit": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/object-visit/-/object-visit-1.0.1.tgz", + "integrity": "sha512-GBaMwwAVK9qbQN3Scdo0OyvgPW7l3lnaVMj84uTOZlswkX0KpF6fyDBJhtTthf7pymztoN36/KEr1DyhF96zEA==", + "requires": { + "isobject": "^3.0.0" + } + }, + "object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + } + }, + "object.getownpropertydescriptors": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.3.tgz", + "integrity": "sha512-VdDoCwvJI4QdC6ndjpqFmoL3/+HxffFBbcJzKi5hwLLqqx3mdbedRpfZDdK0SrOSauj8X4GzBvnDZl4vTN7dOw==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1" + } + }, + "object.omit": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/object.omit/-/object.omit-2.0.1.tgz", + "integrity": "sha512-UiAM5mhmIuKLsOvrL+B0U2d1hXHF3bFYWIuH1LMpuV2EJEHG1Ntz06PgLEHjm6VFd87NpH8rastvPoyv6UW2fA==", + "dev": true, + "requires": { + "for-own": "^0.1.4", + "is-extendable": "^0.1.1" + } + }, + "object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha512-tqa/UMy/CCoYmj+H5qc07qvSL9dqcs/WZENZ1JbtWBlATP+iVOe778gE6MSijnyCnORzDuX6hU+LA4SZ09YjFQ==", + "requires": { + "isobject": "^3.0.1" + } + }, + "object.values": { + "version": "1.1.5", + "resolved": "https://registry.npmmirror.com/object.values/-/object.values-1.1.5.tgz", + "integrity": "sha512-QUZRW0ilQ3PnPpbNtgdNV1PDbEqLIiSFB3l+EnGtBQ/8SUTLj1PZwtQHABZtLgwpJZTSZhuGLOGk57Drx2IvYg==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1" + } + }, + "obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "dev": true + }, + "on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmmirror.com/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "requires": { + "ee-first": "1.1.1" + } + }, + "on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", + "dev": true, + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "open": { + "version": "6.4.0", + "resolved": "https://registry.npmmirror.com/open/-/open-6.4.0.tgz", + "integrity": "sha512-IFenVPgF70fSm1keSd2iDBIDIBZkroLeuffXq+wKTzTJlBpesFWojV9lb8mzOfaAzM1sr7HQHuO0vtV0zYekGg==", + "dev": true, + "requires": { + "is-wsl": "^1.1.0" + } + }, + "opener": { + "version": "1.5.2", + "resolved": "https://registry.npmmirror.com/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", + "dev": true + }, + "opn": { + "version": "5.5.0", + "resolved": "https://registry.npmmirror.com/opn/-/opn-5.5.0.tgz", + "integrity": "sha512-PqHpggC9bLV0VeWcdKhkpxY+3JTzetLSqTCWL/z/tFIbI6G8JCjondXklT1JinczLz2Xib62sSp0T/gKT4KksA==", + "dev": true, + "requires": { + "is-wsl": "^1.1.0" + } + }, + "optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmmirror.com/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dev": true, + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + } + }, + "ora": { + "version": "3.4.0", + "resolved": "https://registry.npmmirror.com/ora/-/ora-3.4.0.tgz", + "integrity": "sha512-eNwHudNbO1folBP3JsZ19v9azXWtQZjICdr3Q0TDPIaeBQ3mXLrh54wM+er0+hSp+dWKf+Z8KM58CYzEyIYxYg==", + "dev": true, + "requires": { + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-spinners": "^2.0.0", + "log-symbols": "^2.2.0", + "strip-ansi": "^5.2.0", + "wcwidth": "^1.0.1" + } + }, + "original": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/original/-/original-1.0.2.tgz", + "integrity": "sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg==", + "dev": true, + "requires": { + "url-parse": "^1.4.3" + } + }, + "os-browserify": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/os-browserify/-/os-browserify-0.3.0.tgz", + "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==" + }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==", + "dev": true + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "dev": true + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmmirror.com/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "dev": true, + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "requires": { + "p-limit": "^2.0.0" + } + }, + "p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", + "dev": true + }, + "p-map-series": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/p-map-series/-/p-map-series-2.1.0.tgz", + "integrity": "sha512-RpYIIK1zXSNEOdwxcfe7FdvGcs7+y5n8rifMhMNWvaxRNMPINJHF5GDeuVxWqnfrcHPSCnp7Oo5yNXHId9Av2Q==", + "dev": true + }, + "p-pipe": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/p-pipe/-/p-pipe-3.1.0.tgz", + "integrity": "sha512-08pj8ATpzMR0Y80x50yJHn37NF6vjrqHutASaX5LiH5npS9XPvrUmscd9MF5R4fuYRHOxQR1FfMIlF7AzwoPqw==", + "dev": true + }, + "p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmmirror.com/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "requires": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + } + }, + "p-reduce": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/p-reduce/-/p-reduce-2.1.0.tgz", + "integrity": "sha512-2USApvnsutq8uoxZBGbbWM0JIYLiEMJ9RlaN7fAzVNb9OZN0SHjjTTfIcb667XynS5Y1VhwDJVDa72TnPzAYWw==", + "dev": true + }, + "p-retry": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/p-retry/-/p-retry-3.0.1.tgz", + "integrity": "sha512-XE6G4+YTTkT2a0UWb2kjZe8xNwf8bIbnqpc/IS/idOBVhyves0mK5OJgeocjx7q5pvX/6m23xuzVPYT1uGM73w==", + "dev": true, + "requires": { + "retry": "^0.12.0" + } + }, + "p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "requires": { + "p-finally": "^1.0.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, + "p-waterfall": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/p-waterfall/-/p-waterfall-2.1.1.tgz", + "integrity": "sha512-RRTnDb2TBG/epPRI2yYXsimO0v3BXC8Yd3ogr1545IaqKK17VGhbWVeGGN+XfCm/08OK8635nH31c8bATkHuSw==", + "dev": true, + "requires": { + "p-reduce": "^2.0.0" + } + }, + "pacote": { + "version": "11.3.5", + "resolved": "https://registry.npmmirror.com/pacote/-/pacote-11.3.5.tgz", + "integrity": "sha512-fT375Yczn4zi+6Hkk2TBe1x1sP8FgFsEIZ2/iWaXY2r/NkhDJfxbcn5paz1+RTFCyNf+dPnaoBDJoAxXSU8Bkg==", + "dev": true, + "requires": { + "@npmcli/git": "^2.1.0", + "@npmcli/installed-package-contents": "^1.0.6", + "@npmcli/promise-spawn": "^1.2.0", + "@npmcli/run-script": "^1.8.2", + "cacache": "^15.0.5", + "chownr": "^2.0.0", + "fs-minipass": "^2.1.0", + "infer-owner": "^1.0.4", + "minipass": "^3.1.3", + "mkdirp": "^1.0.3", + "npm-package-arg": "^8.0.1", + "npm-packlist": "^2.1.4", + "npm-pick-manifest": "^6.0.0", + "npm-registry-fetch": "^11.0.0", + "promise-retry": "^2.0.1", + "read-package-json-fast": "^2.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.1.0" + }, + "dependencies": { + "cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmmirror.com/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "dev": true, + "requires": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + } + }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minipass": { + "version": "3.1.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.1.6.tgz", + "integrity": "sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "dev": true, + "requires": { + "minipass": "^3.1.1" + } + }, + "tar": { + "version": "6.1.11", + "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmmirror.com/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "parallel-transform": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/parallel-transform/-/parallel-transform-1.2.0.tgz", + "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==", + "requires": { + "cyclist": "^1.0.1", + "inherits": "^2.0.3", + "readable-stream": "^2.1.5" + } + }, + "param-case": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/param-case/-/param-case-2.1.1.tgz", + "integrity": "sha512-eQE845L6ot89sk2N8liD8HAuH4ca6Vvr7VWAWwt7+kvvG5aBcPmmphQ68JsEG2qa9n1TykS2DLeMt363AAH8/w==", + "dev": true, + "requires": { + "no-case": "^2.2.0" + } + }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "requires": { + "callsites": "^3.0.0" + }, + "dependencies": { + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + } + } + }, + "parse-asn1": { + "version": "5.1.6", + "resolved": "https://registry.npmmirror.com/parse-asn1/-/parse-asn1-5.1.6.tgz", + "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", + "requires": { + "asn1.js": "^5.2.0", + "browserify-aes": "^1.0.0", + "evp_bytestokey": "^1.0.0", + "pbkdf2": "^3.0.3", + "safe-buffer": "^5.1.1" + } + }, + "parse-glob": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/parse-glob/-/parse-glob-3.0.4.tgz", + "integrity": "sha512-FC5TeK0AwXzq3tUBFtH74naWkPQCEWs4K+xMxWZBlKDWu0bVHXGZa+KKqxKidd7xwhdZ19ZNuF2uO1M/r196HA==", + "dev": true, + "requires": { + "glob-base": "^0.3.0", + "is-dotfile": "^1.0.0", + "is-extglob": "^1.0.0", + "is-glob": "^2.0.0" + }, + "dependencies": { + "is-extglob": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-extglob/-/is-extglob-1.0.0.tgz", + "integrity": "sha512-7Q+VbVafe6x2T+Tu6NcOf6sRklazEPmBoB3IWk3WdGZM2iGUwU/Oe3Wtq5lSEkDTTlpp8yx+5t4pzO/i9Ty1ww==", + "dev": true + }, + "is-glob": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-2.0.1.tgz", + "integrity": "sha512-a1dBeB19NXsf/E0+FHqkagizel/LQw2DjSQpvQrj3zT+jYPpaUCryPnrQajXKFLCMuf4I6FhRpaGtw4lPrG6Eg==", + "dev": true, + "requires": { + "is-extglob": "^1.0.0" + } + } + } + }, + "parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==", + "dev": true, + "requires": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + } + }, + "parse-url": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/parse-url/-/parse-url-6.0.2.tgz", + "integrity": "sha512-uCSjOvD3T+6B/sPWhR+QowAZcU/o4bjPrVBQBGFxcDF6J6FraCGIaDBsdoQawiaaAVdHvtqBe3w3vKlfBKySOQ==", + "dev": true, + "requires": { + "is-ssh": "^1.3.0", + "normalize-url": "^6.1.0", + "parse-path": "^4.0.4", + "protocols": "^1.4.0" + }, + "dependencies": { + "normalize-url": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", + "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", + "dev": true + }, + "parse-path": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/parse-path/-/parse-path-4.0.4.tgz", + "integrity": "sha512-Z2lWUis7jlmXC1jeOG9giRO2+FsuyNipeQ43HAjqAZjwSe3SEf+q/84FGPHoso3kyntbxa4c4i77t3m6fGf8cw==", + "dev": true, + "requires": { + "is-ssh": "^1.3.0", + "protocols": "^1.4.0", + "qs": "^6.9.4", + "query-string": "^6.13.8" + } + }, + "query-string": { + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.14.1.tgz", + "integrity": "sha512-XDxAeVmpfu1/6IjyT/gXHOl+S0vQ9owggJ30hhWKdHAsNPOcasn5o9BW0eejZqL2e4vMjhAxoW3jVHcD6mbcYw==", + "dev": true, + "requires": { + "decode-uri-component": "^0.2.0", + "filter-obj": "^1.1.0", + "split-on-first": "^1.0.0", + "strict-uri-encode": "^2.0.0" + } + }, + "strict-uri-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==", + "dev": true + } + } + }, + "parse5": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/parse5/-/parse5-5.1.1.tgz", + "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==", + "dev": true + }, + "parse5-htmlparser2-tree-adapter": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz", + "integrity": "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==", + "dev": true, + "requires": { + "parse5": "^6.0.1" + }, + "dependencies": { + "parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "dev": true + } + } + }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true + }, + "pascalcase": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/pascalcase/-/pascalcase-0.1.1.tgz", + "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==" + }, + "patch-package": { + "version": "6.2.2", + "resolved": "https://registry.npmmirror.com/patch-package/-/patch-package-6.2.2.tgz", + "integrity": "sha512-YqScVYkVcClUY0v8fF0kWOjDYopzIM8e3bj/RU1DPeEF14+dCGm6UeOYm4jvCyxqIEQ5/eJzmbWfDWnUleFNMg==", + "dev": true, + "requires": { + "@yarnpkg/lockfile": "^1.1.0", + "chalk": "^2.4.2", + "cross-spawn": "^6.0.5", + "find-yarn-workspace-root": "^1.2.1", + "fs-extra": "^7.0.1", + "is-ci": "^2.0.0", + "klaw-sync": "^6.0.0", + "minimist": "^1.2.0", + "rimraf": "^2.6.3", + "semver": "^5.6.0", + "slash": "^2.0.0", + "tmp": "^0.0.33" + }, + "dependencies": { + "ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "dev": true + }, + "is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dev": true, + "requires": { + "ci-info": "^2.0.0" + } + } + } + }, + "path-browserify": { + "version": "0.0.1", + "resolved": "https://registry.npmmirror.com/path-browserify/-/path-browserify-0.0.1.tgz", + "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==" + }, + "path-dirname": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/path-dirname/-/path-dirname-1.0.2.tgz", + "integrity": "sha512-ALzNPpyNq9AqXMBjeymIjFDAkAFH06mHJH/cSBHAgU0s4vfpBn6b2nf8tiRLvagKD8RbTpq2FKTBg7cl9l3c7Q==" + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==" + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", + "dev": true + }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "dev": true + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==", + "dev": true + }, + "path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "requires": { + "pify": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true + } + } + }, + "pbkdf2": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/pbkdf2/-/pbkdf2-3.1.2.tgz", + "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", + "requires": { + "create-hash": "^1.1.2", + "create-hmac": "^1.1.4", + "ripemd160": "^2.0.1", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==", + "dev": true + }, + "picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmmirror.com/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==", + "dev": true + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "dev": true, + "requires": { + "pinkie": "^2.0.0" + } + }, + "pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "requires": { + "find-up": "^3.0.0" + } + }, + "pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/pkg-up/-/pkg-up-2.0.0.tgz", + "integrity": "sha512-fjAPuiws93rm7mPUu21RdBnkeZNrbfCFCwfAhPWY+rR3zG0ubpe5cEReHOw5fIbfmsxEV/g2kSxGTATY3Bpnwg==", + "dev": true, + "requires": { + "find-up": "^2.1.0" + }, + "dependencies": { + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", + "dev": true + } + } + }, + "please-upgrade-node": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz", + "integrity": "sha512-gQR3WpIgNIKwBMVLkpMUeR3e1/E1y42bqDQZfql+kDeXd8COYfM8PQA4X6y7a8u9Ua9FHmsrrmirW2vHs45hWg==", + "dev": true, + "requires": { + "semver-compare": "^1.0.0" + } + }, + "pluralize": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/pluralize/-/pluralize-7.0.0.tgz", + "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==", + "dev": true, + "optional": true + }, + "popper.js": { + "version": "1.16.1", + "resolved": "https://registry.npmmirror.com/popper.js/-/popper.js-1.16.1.tgz", + "integrity": "sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ==" + }, + "portfinder": { + "version": "1.0.28", + "resolved": "https://registry.npmmirror.com/portfinder/-/portfinder-1.0.28.tgz", + "integrity": "sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA==", + "dev": true, + "requires": { + "async": "^2.6.2", + "debug": "^3.1.1", + "mkdirp": "^0.5.5" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } + }, + "posix-character-classes": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz", + "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==" + }, + "postcss": { + "version": "7.0.39", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-7.0.39.tgz", + "integrity": "sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA==", + "dev": true, + "requires": { + "picocolors": "^0.2.1", + "source-map": "^0.6.1" + }, + "dependencies": { + "picocolors": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/picocolors/-/picocolors-0.2.1.tgz", + "integrity": "sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA==", + "dev": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "postcss-calc": { + "version": "7.0.5", + "resolved": "https://registry.npmmirror.com/postcss-calc/-/postcss-calc-7.0.5.tgz", + "integrity": "sha512-1tKHutbGtLtEZF6PT4JSihCHfIVldU72mZ8SdZHIYriIZ9fh9k9aWSppaT8rHsyI3dX+KSR+W+Ix9BMY3AODrg==", + "dev": true, + "requires": { + "postcss": "^7.0.27", + "postcss-selector-parser": "^6.0.2", + "postcss-value-parser": "^4.0.2" + } + }, + "postcss-colormin": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/postcss-colormin/-/postcss-colormin-4.0.3.tgz", + "integrity": "sha512-WyQFAdDZpExQh32j0U0feWisZ0dmOtPl44qYmJKkq9xFWY3p+4qnRzCHeNrkeRhwPHz9bQ3mo0/yVkaply0MNw==", + "dev": true, + "requires": { + "browserslist": "^4.0.0", + "color": "^3.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-convert-values": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/postcss-convert-values/-/postcss-convert-values-4.0.1.tgz", + "integrity": "sha512-Kisdo1y77KUC0Jmn0OXU/COOJbzM8cImvw1ZFsBgBgMgb1iL23Zs/LXRe3r+EZqM3vGYKdQ2YJVQ5VkJI+zEJQ==", + "dev": true, + "requires": { + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-discard-comments": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-discard-comments/-/postcss-discard-comments-4.0.2.tgz", + "integrity": "sha512-RJutN259iuRf3IW7GZyLM5Sw4GLTOH8FmsXBnv8Ab/Tc2k4SR4qbV4DNbyyY4+Sjo362SyDmW2DQ7lBSChrpkg==", + "dev": true, + "requires": { + "postcss": "^7.0.0" + } + }, + "postcss-discard-duplicates": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-discard-duplicates/-/postcss-discard-duplicates-4.0.2.tgz", + "integrity": "sha512-ZNQfR1gPNAiXZhgENFfEglF93pciw0WxMkJeVmw8eF+JZBbMD7jp6C67GqJAXVZP2BWbOztKfbsdmMp/k8c6oQ==", + "dev": true, + "requires": { + "postcss": "^7.0.0" + } + }, + "postcss-discard-empty": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/postcss-discard-empty/-/postcss-discard-empty-4.0.1.tgz", + "integrity": "sha512-B9miTzbznhDjTfjvipfHoqbWKwd0Mj+/fL5s1QOz06wufguil+Xheo4XpOnc4NqKYBCNqqEzgPv2aPBIJLox0w==", + "dev": true, + "requires": { + "postcss": "^7.0.0" + } + }, + "postcss-discard-overridden": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/postcss-discard-overridden/-/postcss-discard-overridden-4.0.1.tgz", + "integrity": "sha512-IYY2bEDD7g1XM1IDEsUT4//iEYCxAmP5oDSFMVU/JVvT7gh+l4fmjciLqGgwjdWpQIdb0Che2VX00QObS5+cTg==", + "dev": true, + "requires": { + "postcss": "^7.0.0" + } + }, + "postcss-load-config": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/postcss-load-config/-/postcss-load-config-2.1.2.tgz", + "integrity": "sha512-/rDeGV6vMUo3mwJZmeHfEDvwnTKKqQ0S7OHUi/kJvvtx3aWtyWG2/0ZWnzCt2keEclwN6Tf0DST2v9kITdOKYw==", + "dev": true, + "requires": { + "cosmiconfig": "^5.0.0", + "import-cwd": "^2.0.0" + } + }, + "postcss-loader": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/postcss-loader/-/postcss-loader-3.0.0.tgz", + "integrity": "sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==", + "dev": true, + "requires": { + "loader-utils": "^1.1.0", + "postcss": "^7.0.0", + "postcss-load-config": "^2.0.0", + "schema-utils": "^1.0.0" + } + }, + "postcss-merge-longhand": { + "version": "4.0.11", + "resolved": "https://registry.npmmirror.com/postcss-merge-longhand/-/postcss-merge-longhand-4.0.11.tgz", + "integrity": "sha512-alx/zmoeXvJjp7L4mxEMjh8lxVlDFX1gqWHzaaQewwMZiVhLo42TEClKaeHbRf6J7j82ZOdTJ808RtN0ZOZwvw==", + "dev": true, + "requires": { + "css-color-names": "0.0.4", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0", + "stylehacks": "^4.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-merge-rules": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/postcss-merge-rules/-/postcss-merge-rules-4.0.3.tgz", + "integrity": "sha512-U7e3r1SbvYzO0Jr3UT/zKBVgYYyhAz0aitvGIYOYK5CPmkNih+WDSsS5tvPrJ8YMQYlEMvsZIiqmn7HdFUaeEQ==", + "dev": true, + "requires": { + "browserslist": "^4.0.0", + "caniuse-api": "^3.0.0", + "cssnano-util-same-parent": "^4.0.0", + "postcss": "^7.0.0", + "postcss-selector-parser": "^3.0.0", + "vendors": "^1.0.0" + }, + "dependencies": { + "postcss-selector-parser": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/postcss-selector-parser/-/postcss-selector-parser-3.1.2.tgz", + "integrity": "sha512-h7fJ/5uWuRVyOtkO45pnt1Ih40CEleeyCHzipqAZO2e5H20g25Y48uYnFUiShvY4rZWNJ/Bib/KVPmanaCtOhA==", + "dev": true, + "requires": { + "dot-prop": "^5.2.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + } + } + } + }, + "postcss-minify-font-values": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-minify-font-values/-/postcss-minify-font-values-4.0.2.tgz", + "integrity": "sha512-j85oO6OnRU9zPf04+PZv1LYIYOprWm6IA6zkXkrJXyRveDEuQggG6tvoy8ir8ZwjLxLuGfNkCZEQG7zan+Hbtg==", + "dev": true, + "requires": { + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-minify-gradients": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-minify-gradients/-/postcss-minify-gradients-4.0.2.tgz", + "integrity": "sha512-qKPfwlONdcf/AndP1U8SJ/uzIJtowHlMaSioKzebAXSG4iJthlWC9iSWznQcX4f66gIWX44RSA841HTHj3wK+Q==", + "dev": true, + "requires": { + "cssnano-util-get-arguments": "^4.0.0", + "is-color-stop": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-minify-params": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-minify-params/-/postcss-minify-params-4.0.2.tgz", + "integrity": "sha512-G7eWyzEx0xL4/wiBBJxJOz48zAKV2WG3iZOqVhPet/9geefm/Px5uo1fzlHu+DOjT+m0Mmiz3jkQzVHe6wxAWg==", + "dev": true, + "requires": { + "alphanum-sort": "^1.0.0", + "browserslist": "^4.0.0", + "cssnano-util-get-arguments": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0", + "uniqs": "^2.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-minify-selectors": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-minify-selectors/-/postcss-minify-selectors-4.0.2.tgz", + "integrity": "sha512-D5S1iViljXBj9kflQo4YutWnJmwm8VvIsU1GeXJGiG9j8CIg9zs4voPMdQDUmIxetUOh60VilsNzCiAFTOqu3g==", + "dev": true, + "requires": { + "alphanum-sort": "^1.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-selector-parser": "^3.0.0" + }, + "dependencies": { + "postcss-selector-parser": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/postcss-selector-parser/-/postcss-selector-parser-3.1.2.tgz", + "integrity": "sha512-h7fJ/5uWuRVyOtkO45pnt1Ih40CEleeyCHzipqAZO2e5H20g25Y48uYnFUiShvY4rZWNJ/Bib/KVPmanaCtOhA==", + "dev": true, + "requires": { + "dot-prop": "^5.2.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + } + } + } + }, + "postcss-modules-extract-imports": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-1.2.1.tgz", + "integrity": "sha512-6jt9XZwUhwmRUhb/CkyJY020PYaPJsCyt3UjbaWo6XEbH/94Hmv6MP7fG2C5NDU/BcHzyGYxNtHvM+LTf9HrYw==", + "dev": true, + "requires": { + "postcss": "^6.0.1" + }, + "dependencies": { + "postcss": { + "version": "6.0.23", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-6.0.23.tgz", + "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==", + "dev": true, + "requires": { + "chalk": "^2.4.1", + "source-map": "^0.6.1", + "supports-color": "^5.4.0" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "postcss-modules-local-by-default": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-1.2.0.tgz", + "integrity": "sha512-X4cquUPIaAd86raVrBwO8fwRfkIdbwFu7CTfEOjiZQHVQwlHRSkTgH5NLDmMm5+1hQO8u6dZ+TOOJDbay1hYpA==", + "dev": true, + "requires": { + "css-selector-tokenizer": "^0.7.0", + "postcss": "^6.0.1" + }, + "dependencies": { + "postcss": { + "version": "6.0.23", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-6.0.23.tgz", + "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==", + "dev": true, + "requires": { + "chalk": "^2.4.1", + "source-map": "^0.6.1", + "supports-color": "^5.4.0" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "postcss-modules-scope": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/postcss-modules-scope/-/postcss-modules-scope-1.1.0.tgz", + "integrity": "sha512-LTYwnA4C1He1BKZXIx1CYiHixdSe9LWYVKadq9lK5aCCMkoOkFyZ7aigt+srfjlRplJY3gIol6KUNefdMQJdlw==", + "dev": true, + "requires": { + "css-selector-tokenizer": "^0.7.0", + "postcss": "^6.0.1" + }, + "dependencies": { + "postcss": { + "version": "6.0.23", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-6.0.23.tgz", + "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==", + "dev": true, + "requires": { + "chalk": "^2.4.1", + "source-map": "^0.6.1", + "supports-color": "^5.4.0" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "postcss-modules-values": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/postcss-modules-values/-/postcss-modules-values-1.3.0.tgz", + "integrity": "sha512-i7IFaR9hlQ6/0UgFuqM6YWaCfA1Ej8WMg8A5DggnH1UGKJvTV/ugqq/KaULixzzOi3T/tF6ClBXcHGCzdd5unA==", + "dev": true, + "requires": { + "icss-replace-symbols": "^1.1.0", + "postcss": "^6.0.1" + }, + "dependencies": { + "postcss": { + "version": "6.0.23", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-6.0.23.tgz", + "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==", + "dev": true, + "requires": { + "chalk": "^2.4.1", + "source-map": "^0.6.1", + "supports-color": "^5.4.0" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "postcss-normalize-charset": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/postcss-normalize-charset/-/postcss-normalize-charset-4.0.1.tgz", + "integrity": "sha512-gMXCrrlWh6G27U0hF3vNvR3w8I1s2wOBILvA87iNXaPvSNo5uZAMYsZG7XjCUf1eVxuPfyL4TJ7++SGZLc9A3g==", + "dev": true, + "requires": { + "postcss": "^7.0.0" + } + }, + "postcss-normalize-display-values": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-normalize-display-values/-/postcss-normalize-display-values-4.0.2.tgz", + "integrity": "sha512-3F2jcsaMW7+VtRMAqf/3m4cPFhPD3EFRgNs18u+k3lTJJlVe7d0YPO+bnwqo2xg8YiRpDXJI2u8A0wqJxMsQuQ==", + "dev": true, + "requires": { + "cssnano-util-get-match": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-normalize-positions": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-normalize-positions/-/postcss-normalize-positions-4.0.2.tgz", + "integrity": "sha512-Dlf3/9AxpxE+NF1fJxYDeggi5WwV35MXGFnnoccP/9qDtFrTArZ0D0R+iKcg5WsUd8nUYMIl8yXDCtcrT8JrdA==", + "dev": true, + "requires": { + "cssnano-util-get-arguments": "^4.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-normalize-repeat-style": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-4.0.2.tgz", + "integrity": "sha512-qvigdYYMpSuoFs3Is/f5nHdRLJN/ITA7huIoCyqqENJe9PvPmLhNLMu7QTjPdtnVf6OcYYO5SHonx4+fbJE1+Q==", + "dev": true, + "requires": { + "cssnano-util-get-arguments": "^4.0.0", + "cssnano-util-get-match": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-normalize-string": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-normalize-string/-/postcss-normalize-string-4.0.2.tgz", + "integrity": "sha512-RrERod97Dnwqq49WNz8qo66ps0swYZDSb6rM57kN2J+aoyEAJfZ6bMx0sx/F9TIEX0xthPGCmeyiam/jXif0eA==", + "dev": true, + "requires": { + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-normalize-timing-functions": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-4.0.2.tgz", + "integrity": "sha512-acwJY95edP762e++00Ehq9L4sZCEcOPyaHwoaFOhIwWCDfik6YvqsYNxckee65JHLKzuNSSmAdxwD2Cud1Z54A==", + "dev": true, + "requires": { + "cssnano-util-get-match": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-normalize-unicode": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/postcss-normalize-unicode/-/postcss-normalize-unicode-4.0.1.tgz", + "integrity": "sha512-od18Uq2wCYn+vZ/qCOeutvHjB5jm57ToxRaMeNuf0nWVHaP9Hua56QyMF6fs/4FSUnVIw0CBPsU0K4LnBPwYwg==", + "dev": true, + "requires": { + "browserslist": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-normalize-url": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/postcss-normalize-url/-/postcss-normalize-url-4.0.1.tgz", + "integrity": "sha512-p5oVaF4+IHwu7VpMan/SSpmpYxcJMtkGppYf0VbdH5B6hN8YNmVyJLuY9FmLQTzY3fag5ESUUHDqM+heid0UVA==", + "dev": true, + "requires": { + "is-absolute-url": "^2.0.0", + "normalize-url": "^3.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-normalize-whitespace": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-normalize-whitespace/-/postcss-normalize-whitespace-4.0.2.tgz", + "integrity": "sha512-tO8QIgrsI3p95r8fyqKV+ufKlSHh9hMJqACqbv2XknufqEDhDvbguXGBBqxw9nsQoXWf0qOqppziKJKHMD4GtA==", + "dev": true, + "requires": { + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-ordered-values": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/postcss-ordered-values/-/postcss-ordered-values-4.1.2.tgz", + "integrity": "sha512-2fCObh5UanxvSxeXrtLtlwVThBvHn6MQcu4ksNT2tsaV2Fg76R2CV98W7wNSlX+5/pFwEyaDwKLLoEV7uRybAw==", + "dev": true, + "requires": { + "cssnano-util-get-arguments": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-prefix-selector": { + "version": "1.15.0", + "resolved": "https://registry.npmmirror.com/postcss-prefix-selector/-/postcss-prefix-selector-1.15.0.tgz", + "integrity": "sha512-9taaTPs6I4906QC03zBBt0LfTWAhrqEWlKSj0jRlxrg1yV+O91h0wcquu6krcA5L6aEv3QnCeG8B1vZ5WT4ecQ==", + "dev": true + }, + "postcss-reduce-initial": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/postcss-reduce-initial/-/postcss-reduce-initial-4.0.3.tgz", + "integrity": "sha512-gKWmR5aUulSjbzOfD9AlJiHCGH6AEVLaM0AV+aSioxUDd16qXP1PCh8d1/BGVvpdWn8k/HiK7n6TjeoXN1F7DA==", + "dev": true, + "requires": { + "browserslist": "^4.0.0", + "caniuse-api": "^3.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0" + } + }, + "postcss-reduce-transforms": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/postcss-reduce-transforms/-/postcss-reduce-transforms-4.0.2.tgz", + "integrity": "sha512-EEVig1Q2QJ4ELpJXMZR8Vt5DQx8/mo+dGWSR7vWXqcob2gQLyQGsionYcGKATXvQzMPn6DSN1vTN7yFximdIAg==", + "dev": true, + "requires": { + "cssnano-util-get-match": "^4.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-selector-parser": { + "version": "6.0.10", + "resolved": "https://registry.npmmirror.com/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz", + "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==", + "dev": true, + "requires": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + } + }, + "postcss-svgo": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/postcss-svgo/-/postcss-svgo-4.0.3.tgz", + "integrity": "sha512-NoRbrcMWTtUghzuKSoIm6XV+sJdvZ7GZSc3wdBN0W19FTtp2ko8NqLsgoh/m9CzNhU3KLPvQmjIwtaNFkaFTvw==", + "dev": true, + "requires": { + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0", + "svgo": "^1.0.0" + }, + "dependencies": { + "postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", + "dev": true + } + } + }, + "postcss-unique-selectors": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/postcss-unique-selectors/-/postcss-unique-selectors-4.0.1.tgz", + "integrity": "sha512-+JanVaryLo9QwZjKrmJgkI4Fn8SBgRO6WXQBJi7KiAVPlmxikB5Jzc4EvXMT2H0/m0RjrVVm9rGNhZddm/8Spg==", + "dev": true, + "requires": { + "alphanum-sort": "^1.0.0", + "postcss": "^7.0.0", + "uniqs": "^2.0.0" + } + }, + "postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true + }, + "posthtml": { + "version": "0.9.2", + "resolved": "https://registry.npmmirror.com/posthtml/-/posthtml-0.9.2.tgz", + "integrity": "sha512-spBB5sgC4cv2YcW03f/IAUN1pgDJWNWD8FzkyY4mArLUMJW+KlQhlmUdKAHQuPfb00Jl5xIfImeOsf6YL8QK7Q==", + "dev": true, + "requires": { + "posthtml-parser": "^0.2.0", + "posthtml-render": "^1.0.5" + } + }, + "posthtml-parser": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/posthtml-parser/-/posthtml-parser-0.2.1.tgz", + "integrity": "sha512-nPC53YMqJnc/+1x4fRYFfm81KV2V+G9NZY+hTohpYg64Ay7NemWWcV4UWuy/SgMupqQ3kJ88M/iRfZmSnxT+pw==", + "dev": true, + "requires": { + "htmlparser2": "^3.8.3", + "isobject": "^2.1.0" + }, + "dependencies": { + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmmirror.com/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dev": true, + "requires": { + "domelementtype": "1" + } + }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "dev": true + }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmmirror.com/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "dev": true, + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + } + }, + "isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "dev": true, + "requires": { + "isarray": "1.0.0" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "posthtml-rename-id": { + "version": "1.0.12", + "resolved": "https://registry.npmmirror.com/posthtml-rename-id/-/posthtml-rename-id-1.0.12.tgz", + "integrity": "sha512-UKXf9OF/no8WZo9edRzvuMenb6AD5hDLzIepJW+a4oJT+T/Lx7vfMYWT4aWlGNQh0WMhnUx1ipN9OkZ9q+ddEw==", + "dev": true, + "requires": { + "escape-string-regexp": "1.0.5" + } + }, + "posthtml-render": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/posthtml-render/-/posthtml-render-1.4.0.tgz", + "integrity": "sha512-W1779iVHGfq0Fvh2PROhCe2QhB8mEErgqzo1wpIt36tCgChafP+hbXIhLDOM8ePJrZcFs0vkNEtdibEWVqChqw==", + "dev": true + }, + "posthtml-svg-mode": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/posthtml-svg-mode/-/posthtml-svg-mode-1.0.3.tgz", + "integrity": "sha512-hEqw9NHZ9YgJ2/0G7CECOeuLQKZi8HjWLkBaSVtOWjygQ9ZD8P7tqeowYs7WrFdKsWEKG7o+IlsPY8jrr0CJpQ==", + "dev": true, + "requires": { + "merge-options": "1.0.1", + "posthtml": "^0.9.2", + "posthtml-parser": "^0.2.1", + "posthtml-render": "^1.0.6" + } + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "dev": true + }, + "prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha512-PhmXi5XmoyKw1Un4E+opM2KcsJInDvKyuOumcjjw3waw86ZNjHwVUOOWLc4bCzLdcKNaWBH9e99sbWzDQsVaYg==", + "dev": true + }, + "preserve": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/preserve/-/preserve-0.2.0.tgz", + "integrity": "sha512-s/46sYeylUfHNjI+sA/78FAHlmIuKqI9wNnzEOGehAlUUYeObv5C2mOinXBjyUyWmJ2SfcS2/ydApH4hTF4WXQ==", + "dev": true + }, + "prettier": { + "version": "2.6.2", + "resolved": "https://registry.npmmirror.com/prettier/-/prettier-2.6.2.tgz", + "integrity": "sha512-PkUpF+qoXTqhOeWL9fu7As8LXsIUZ1WYaJiY/a7McAQzxjk82OF0tibkFXVCDImZtWxbvojFjerkiLb0/q8mew==", + "dev": true, + "optional": true + }, + "pretty-error": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/pretty-error/-/pretty-error-2.1.2.tgz", + "integrity": "sha512-EY5oDzmsX5wvuynAByrmY0P0hcp+QpnAKbJng2A2MPjVKXCxrDSUkzghVJ4ZGPIv+JC4gX8fPUWscC0RtjsWGw==", + "dev": true, + "requires": { + "lodash": "^4.17.20", + "renderkid": "^2.0.4" + } + }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmmirror.com/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true + }, + "promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==" + }, + "promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "dev": true, + "requires": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + } + }, + "promzard": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/promzard/-/promzard-0.3.0.tgz", + "integrity": "sha512-JZeYqd7UAcHCwI+sTOeUDYkvEU+1bQ7iE0UT1MgB/tERkAPkesW46MrpIySzODi+owTjZtiF8Ay5j9m60KmMBw==", + "dev": true, + "requires": { + "read": "1" + } + }, + "property-expr": { + "version": "1.5.1", + "resolved": "https://registry.npmmirror.com/property-expr/-/property-expr-1.5.1.tgz", + "integrity": "sha512-CGuc0VUTGthpJXL36ydB6jnbyOf/rAHFvmVrJlH+Rg0DqqLFQGAP6hIaxD/G0OAmBJPhXDHuEJigrp0e0wFV6g==", + "dev": true + }, + "proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", + "dev": true + }, + "protocols": { + "version": "1.4.8", + "resolved": "https://registry.npmmirror.com/protocols/-/protocols-1.4.8.tgz", + "integrity": "sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg==", + "dev": true + }, + "proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, + "prr": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/prr/-/prr-1.0.1.tgz", + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==" + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==", + "dev": true + }, + "psl": { + "version": "1.8.0", + "resolved": "https://registry.npmmirror.com/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==", + "dev": true + }, + "public-encrypt": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/public-encrypt/-/public-encrypt-4.0.3.tgz", + "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", + "requires": { + "bn.js": "^4.1.0", + "browserify-rsa": "^4.0.0", + "create-hash": "^1.1.0", + "parse-asn1": "^5.0.0", + "randombytes": "^2.0.1", + "safe-buffer": "^5.1.2" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmmirror.com/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "pumpify": { + "version": "1.5.1", + "resolved": "https://registry.npmmirror.com/pumpify/-/pumpify-1.5.1.tgz", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "requires": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" + }, + "dependencies": { + "pump": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } + } + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmmirror.com/q/-/q-1.5.1.tgz", + "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==" + }, + "qs": { + "version": "6.9.4", + "resolved": "https://registry.npmmirror.com/qs/-/qs-6.9.4.tgz", + "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==" + }, + "query-string": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/query-string/-/query-string-4.3.4.tgz", + "integrity": "sha512-O2XLNDBIg1DnTOa+2XrIwSiXEV8h2KImXUnjhhn2+UsvZ+Es2uyd5CCRTNQlDGbzUQOW3aYCBx9rVA6dzsiY7Q==", + "dev": true, + "requires": { + "object-assign": "^4.1.0", + "strict-uri-encode": "^1.0.0" + } + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==" + }, + "querystring-es3": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/querystring-es3/-/querystring-es3-0.2.1.tgz", + "integrity": "sha512-773xhDQnZBMFobEiztv8LIl70ch5MSF/jUQVlhwFyBILqq96anmoctVIYz+ZRp0qbCKATTn6ev02M3r7Ga5vqA==" + }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true + }, + "quick-lru": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/quick-lru/-/quick-lru-4.0.1.tgz", + "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==", + "dev": true + }, + "randomatic": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/randomatic/-/randomatic-3.1.1.tgz", + "integrity": "sha512-TuDE5KxZ0J461RVjrJZCJc+J+zCkTb1MbH9AQUq68sMhOMcy9jLcb3BrZKgp9q9Ncltdg4QVqWrH02W2EFFVYw==", + "dev": true, + "requires": { + "is-number": "^4.0.0", + "kind-of": "^6.0.0", + "math-random": "^1.0.1" + }, + "dependencies": { + "is-number": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/is-number/-/is-number-4.0.0.tgz", + "integrity": "sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ==", + "dev": true + } + } + }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "requires": { + "safe-buffer": "^5.1.0" + } + }, + "randomfill": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/randomfill/-/randomfill-1.0.4.tgz", + "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", + "requires": { + "randombytes": "^2.0.5", + "safe-buffer": "^5.1.0" + } + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true + }, + "raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmmirror.com/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dev": true, + "requires": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, + "read": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/read/-/read-1.0.7.tgz", + "integrity": "sha512-rSOKNYUmaxy0om1BNjMN4ezNT6VKK+2xF4GBhc81mkH7L60i6dp8qPYrkndNLT3QPphoII3maL9PVC9XmhHwVQ==", + "dev": true, + "requires": { + "mute-stream": "~0.0.4" + } + }, + "read-cmd-shim": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/read-cmd-shim/-/read-cmd-shim-2.0.0.tgz", + "integrity": "sha512-HJpV9bQpkl6KwjxlJcBoqu9Ba0PQg8TqSNIOrulGt54a0uup0HtevreFHzYzkm0lpnleRdNBzXznKrgxglEHQw==", + "dev": true + }, + "read-package-json": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/read-package-json/-/read-package-json-2.1.2.tgz", + "integrity": "sha512-D1KmuLQr6ZSJS0tW8hf3WGpRlwszJOXZ3E8Yd/DNRaM5d+1wVRZdHlpGBLAuovjr28LbWvjpWkBHMxpRGGjzNA==", + "dev": true, + "requires": { + "glob": "^7.1.1", + "json-parse-even-better-errors": "^2.3.0", + "normalize-package-data": "^2.0.0", + "npm-normalize-package-bin": "^1.0.0" + } + }, + "read-package-json-fast": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/read-package-json-fast/-/read-package-json-fast-2.0.3.tgz", + "integrity": "sha512-W/BKtbL+dUjTuRL2vziuYhp76s5HZ9qQhd/dKfWIZveD0O40453QNyZhC0e63lqZrAQ4jiOapVoeJ7JrszenQQ==", + "dev": true, + "requires": { + "json-parse-even-better-errors": "^2.3.0", + "npm-normalize-package-bin": "^1.0.1" + } + }, + "read-package-tree": { + "version": "5.3.1", + "resolved": "https://registry.npmmirror.com/read-package-tree/-/read-package-tree-5.3.1.tgz", + "integrity": "sha512-mLUDsD5JVtlZxjSlPPx1RETkNjjvQYuweKwNVt1Sn8kP5Jh44pvYuUHCp6xSVDZWbNxVxG5lyZJ921aJH61sTw==", + "dev": true, + "requires": { + "read-package-json": "^2.0.0", + "readdir-scoped-modules": "^1.0.0", + "util-promisify": "^2.1.0" + } + }, + "read-pkg": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/read-pkg/-/read-pkg-5.2.0.tgz", + "integrity": "sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==", + "dev": true, + "requires": { + "@types/normalize-package-data": "^2.4.0", + "normalize-package-data": "^2.5.0", + "parse-json": "^5.0.0", + "type-fest": "^0.6.0" + }, + "dependencies": { + "parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + } + } + } + }, + "read-pkg-up": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/read-pkg-up/-/read-pkg-up-3.0.0.tgz", + "integrity": "sha512-YFzFrVvpC6frF1sz8psoHDBGF7fLPc+llq/8NB43oagqWkx8ar5zYtsTORtOjw9W2RHLpWP+zTWwBvf1bCmcSw==", + "dev": true, + "requires": { + "find-up": "^2.0.0", + "read-pkg": "^3.0.0" + }, + "dependencies": { + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, + "load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", + "dev": true + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true + }, + "read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha512-BLq/cCO9two+lBgiTYNqD6GdtK8s4NpaWrl6/rCO9w0TUS8oJl7cmToOZfRYllKTISY6nt1U7jQ53brmKqY6BA==", + "dev": true, + "requires": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + } + } + } + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "readdir-scoped-modules": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz", + "integrity": "sha512-asaikDeqAQg7JifRsZn1NJZXo9E+VwlyCfbkZhwyISinqk5zNS6266HS5kah6P0SaQKGF6SkNnZVHUzHFYxYDw==", + "dev": true, + "requires": { + "debuglog": "^1.0.1", + "dezalgo": "^1.0.0", + "graceful-fs": "^4.1.2", + "once": "^1.3.0" + } + }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "optional": true, + "requires": { + "picomatch": "^2.2.1" + } + }, + "redent": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/redent/-/redent-3.0.0.tgz", + "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", + "dev": true, + "requires": { + "indent-string": "^4.0.0", + "strip-indent": "^3.0.0" + }, + "dependencies": { + "strip-indent": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/strip-indent/-/strip-indent-3.0.0.tgz", + "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", + "dev": true, + "requires": { + "min-indent": "^1.0.0" + } + } + } + }, + "regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmmirror.com/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==", + "dev": true + }, + "regenerate-unicode-properties": { + "version": "10.0.1", + "resolved": "https://registry.npmmirror.com/regenerate-unicode-properties/-/regenerate-unicode-properties-10.0.1.tgz", + "integrity": "sha512-vn5DU6yg6h8hP/2OkQo3K7uVILvY4iu0oI4t3HFa81UPkhGJwkRwM10JEc3upjdhHjs/k8GJY1sRBhk5sr69Bw==", + "dev": true, + "requires": { + "regenerate": "^1.4.2" + } + }, + "regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmmirror.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", + "dev": true + }, + "regenerator-transform": { + "version": "0.15.0", + "resolved": "https://registry.npmmirror.com/regenerator-transform/-/regenerator-transform-0.15.0.tgz", + "integrity": "sha512-LsrGtPmbYg19bcPHwdtmXwbW+TqNvtY4riE3P83foeHRroMbH6/2ddFBfab3t7kbzc7v7p4wbkIecHImqt0QNg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.8.4" + } + }, + "regex-cache": { + "version": "0.4.4", + "resolved": "https://registry.npmmirror.com/regex-cache/-/regex-cache-0.4.4.tgz", + "integrity": "sha512-nVIZwtCjkC9YgvWkpM55B5rBhBYRZhAaJbgcFYXXsHnbZ9UZI9nnVWYZpBlCqv9ho2eZryPnWrZGsOdPwVWXWQ==", + "dev": true, + "requires": { + "is-equal-shallow": "^0.1.3" + } + }, + "regex-not": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/regex-not/-/regex-not-1.0.2.tgz", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "requires": { + "extend-shallow": "^3.0.2", + "safe-regex": "^1.1.0" + } + }, + "regexp.prototype.flags": { + "version": "1.4.3", + "resolved": "https://registry.npmmirror.com/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", + "integrity": "sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "functions-have-names": "^1.2.2" + } + }, + "regexpp": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/regexpp/-/regexpp-1.1.0.tgz", + "integrity": "sha512-LOPw8FpgdQF9etWMaAfG/WRthIdXJGYp4mJ2Jgn/2lpkbod9jPn0t9UqN7AxBOKNfzRbYyVfgc7Vk4t/MpnXgw==", + "dev": true, + "optional": true + }, + "regexpu-core": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/regexpu-core/-/regexpu-core-5.0.1.tgz", + "integrity": "sha512-CriEZlrKK9VJw/xQGJpQM5rY88BtuL8DM+AEwvcThHilbxiTAy8vq4iJnd2tqq8wLmjbGZzP7ZcKFjbGkmEFrw==", + "dev": true, + "requires": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.0.1", + "regjsgen": "^0.6.0", + "regjsparser": "^0.8.2", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.0.0" + } + }, + "regjsgen": { + "version": "0.6.0", + "resolved": "https://registry.npmmirror.com/regjsgen/-/regjsgen-0.6.0.tgz", + "integrity": "sha512-ozE883Uigtqj3bx7OhL1KNbCzGyW2NQZPl6Hs09WTvCuZD5sTI4JY58bkbQWa/Y9hxIsvJ3M8Nbf7j54IqeZbA==", + "dev": true + }, + "regjsparser": { + "version": "0.8.4", + "resolved": "https://registry.npmmirror.com/regjsparser/-/regjsparser-0.8.4.tgz", + "integrity": "sha512-J3LABycON/VNEu3abOviqGHuB/LOtOQj8SKmfP9anY5GfAVw/SPjwzSjxGjbZXIxbGfqTHtJw58C2Li/WkStmA==", + "dev": true, + "requires": { + "jsesc": "~0.5.0" + }, + "dependencies": { + "jsesc": { + "version": "0.5.0", + "resolved": "https://registry.npmmirror.com/jsesc/-/jsesc-0.5.0.tgz", + "integrity": "sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==", + "dev": true + } + } + }, + "relateurl": { + "version": "0.2.7", + "resolved": "https://registry.npmmirror.com/relateurl/-/relateurl-0.2.7.tgz", + "integrity": "sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==", + "dev": true + }, + "remove-trailing-separator": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", + "integrity": "sha512-/hS+Y0u3aOfIETiaiirUFwDBDzmXPvO+jAfKTitUngIPzdKc6Z0LoFjM/CK5PL4C+eKwHohlHAb6H0VFfmmUsw==" + }, + "renderkid": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/renderkid/-/renderkid-2.0.7.tgz", + "integrity": "sha512-oCcFyxaMrKsKcTY59qnCAtmDVSLfPbrv6A3tVbPdFMMrv5jaK10V6m40cKsoPNhAqN6rmHW9sswW4o3ruSrwUQ==", + "dev": true, + "requires": { + "css-select": "^4.1.3", + "dom-converter": "^0.2.0", + "htmlparser2": "^6.1.0", + "lodash": "^4.17.21", + "strip-ansi": "^3.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "css-select": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/css-select/-/css-select-4.3.0.tgz", + "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==", + "dev": true, + "requires": { + "boolbase": "^1.0.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", + "domutils": "^2.8.0", + "nth-check": "^2.0.1" + } + }, + "css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true + }, + "dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "dev": true, + "requires": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + } + }, + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true + }, + "domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmmirror.com/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "dev": true, + "requires": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmmirror.com/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "nth-check": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/nth-check/-/nth-check-2.0.1.tgz", + "integrity": "sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==", + "dev": true, + "requires": { + "boolbase": "^1.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + } + } + }, + "repeat-element": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/repeat-element/-/repeat-element-1.1.4.tgz", + "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==" + }, + "repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmmirror.com/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==" + }, + "repeating": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/repeating/-/repeating-2.0.1.tgz", + "integrity": "sha512-ZqtSMuVybkISo2OWvqvm7iHSWngvdaW3IpsT9/uP8v4gMi591LY6h35wdOfvQdWCKFWZWm2Y1Opp4kV7vQKT6A==", + "dev": true, + "requires": { + "is-finite": "^1.0.0" + } + }, + "request": { + "version": "2.88.2", + "resolved": "https://registry.npmmirror.com/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "dev": true, + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "qs": { + "version": "6.5.3", + "resolved": "https://registry.npmmirror.com/qs/-/qs-6.5.3.tgz", + "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==", + "dev": true + } + } + }, + "request-promise-core": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/request-promise-core/-/request-promise-core-1.1.4.tgz", + "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==", + "dev": true, + "requires": { + "lodash": "^4.17.19" + } + }, + "request-promise-native": { + "version": "1.0.9", + "resolved": "https://registry.npmmirror.com/request-promise-native/-/request-promise-native-1.0.9.tgz", + "integrity": "sha512-wcW+sIUiWnKgNY0dqCpOZkUbF/I+YPi+f09JZIDa39Ec+q82CpSYniDp+ISgTTbKmnpJWASeJBPZmoxH84wt3g==", + "dev": true, + "requires": { + "request-promise-core": "1.1.4", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "require-uncached": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/require-uncached/-/require-uncached-1.0.3.tgz", + "integrity": "sha512-Xct+41K3twrbBHdxAgMoOS+cNcoqIjfM2/VxBF4LL2hVph7YsF8VSKyQ3BDFZwEVbok9yeDl2le/qo0S77WG2w==", + "dev": true, + "optional": true, + "requires": { + "caller-path": "^0.1.0", + "resolve-from": "^1.0.0" + } + }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, + "reselect": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/reselect/-/reselect-3.0.1.tgz", + "integrity": "sha512-b/6tFZCmRhtBMa4xGqiiRp9jh9Aqi2A687Lo265cN0/QohJQEBPiQ52f4QB6i0eF3yp3hmLL21LSGBcML2dlxA==", + "dev": true + }, + "resolve": { + "version": "1.22.0", + "resolved": "https://registry.npmmirror.com/resolve/-/resolve-1.22.0.tgz", + "integrity": "sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw==", + "dev": true, + "requires": { + "is-core-module": "^2.8.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "resolve-cwd": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/resolve-cwd/-/resolve-cwd-2.0.0.tgz", + "integrity": "sha512-ccu8zQTrzVr954472aUVPLEcB3YpKSYR3cg/3lo1okzobPBM+1INXBbBZlDbnI/hbEocnf8j0QVo43hQKrbchg==", + "dev": true, + "requires": { + "resolve-from": "^3.0.0" + }, + "dependencies": { + "resolve-from": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-3.0.0.tgz", + "integrity": "sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw==", + "dev": true + } + } + }, + "resolve-from": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-1.0.1.tgz", + "integrity": "sha512-kT10v4dhrlLNcnO084hEjvXCI1wUG9qZLoz2RogxqDQQYy7IxjI/iMUkOtQTNEh6rzHxvdQWHsJyel1pKOVCxg==", + "dev": true, + "optional": true + }, + "resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==" + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", + "dev": true, + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + }, + "ret": { + "version": "0.1.15", + "resolved": "https://registry.npmmirror.com/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" + }, + "retry": { + "version": "0.12.0", + "resolved": "https://registry.npmmirror.com/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true + }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true + }, + "rgb-regex": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/rgb-regex/-/rgb-regex-1.0.1.tgz", + "integrity": "sha512-gDK5mkALDFER2YLqH6imYvK6g02gpNGM4ILDZ472EwWfXZnC2ZEpoB2ECXTyOVUKuk/bPJZMzwQPBYICzP+D3w==", + "dev": true + }, + "rgba-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/rgba-regex/-/rgba-regex-1.0.0.tgz", + "integrity": "sha512-zgn5OjNQXLUTdq8m17KdaicF6w89TZs8ZU8y0AYENIU6wG8GG6LLm0yLSiPY8DmaYmHdgRW8rnApjoT0fQRfMg==", + "dev": true + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + } + }, + "ripemd160": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/ripemd160/-/ripemd160-2.0.2.tgz", + "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", + "requires": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1" + } + }, + "run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmmirror.com/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "dev": true + }, + "run-node": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/run-node/-/run-node-1.0.0.tgz", + "integrity": "sha512-kc120TBlQ3mih1LSzdAJXo4xn/GWS2ec0l3S+syHDXP9uRr0JAT8Qd3mdMuyjqCzeZktgP3try92cEgf9Nks8A==", + "dev": true + }, + "run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "requires": { + "queue-microtask": "^1.2.2" + } + }, + "run-queue": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/run-queue/-/run-queue-1.0.3.tgz", + "integrity": "sha512-ntymy489o0/QQplUDnpYAYUsO50K9SBrIVaKCWDOJzYJts0f9WH9RFJkyagebkw5+y1oi00R7ynNW/d12GBumg==", + "requires": { + "aproba": "^1.1.1" + } + }, + "rx-lite": { + "version": "4.0.8", + "resolved": "https://registry.npmmirror.com/rx-lite/-/rx-lite-4.0.8.tgz", + "integrity": "sha512-Cun9QucwK6MIrp3mry/Y7hqD1oFqTYLQ4pGxaHTjIdaFDWRGGLikqp6u8LcWJnzpoALg9hap+JGk8sFIUuEGNA==", + "dev": true, + "optional": true + }, + "rx-lite-aggregates": { + "version": "4.0.8", + "resolved": "https://registry.npmmirror.com/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz", + "integrity": "sha512-3xPNZGW93oCjiO7PtKxRK6iOVYBWBvtf9QHDfU23Oc+dLIQmAV//UnyXV/yihv81VS/UqoQPk4NegS8EFi55Hg==", + "dev": true, + "optional": true, + "requires": { + "rx-lite": "*" + } + }, + "rxjs": { + "version": "6.6.7", + "resolved": "https://registry.npmmirror.com/rxjs/-/rxjs-6.6.7.tgz", + "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==", + "dev": true, + "requires": { + "tslib": "^1.9.0" + }, + "dependencies": { + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true + } + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "requires": { + "ret": "~0.1.10" + } + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sass-graph": { + "version": "2.2.5", + "resolved": "https://registry.npmmirror.com/sass-graph/-/sass-graph-2.2.5.tgz", + "integrity": "sha512-VFWDAHOe6mRuT4mZRd4eKE+d8Uedrk6Xnh7Sh9b4NGufQLQjOrvf/MQoOdx+0s92L89FeyUUNfU597j/3uNpag==", + "dev": true, + "requires": { + "glob": "^7.0.0", + "lodash": "^4.0.0", + "scss-tokenizer": "^0.2.3", + "yargs": "^13.3.2" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "yargs": { + "version": "13.3.2", + "resolved": "https://registry.npmmirror.com/yargs/-/yargs-13.3.2.tgz", + "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "dev": true, + "requires": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.2" + } + }, + "yargs-parser": { + "version": "13.1.2", + "resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + } + } + }, + "sass-loader": { + "version": "7.3.1", + "resolved": "https://registry.npmmirror.com/sass-loader/-/sass-loader-7.3.1.tgz", + "integrity": "sha512-tuU7+zm0pTCynKYHpdqaPpe+MMTQ76I9TPZ7i4/5dZsigE350shQWe5EZNl5dBidM49TPET75tNqRbcsUZWeNA==", + "dev": true, + "requires": { + "clone-deep": "^4.0.1", + "loader-utils": "^1.0.1", + "neo-async": "^2.5.0", + "pify": "^4.0.1", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, + "schema-utils": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/schema-utils/-/schema-utils-1.0.0.tgz", + "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "requires": { + "ajv": "^6.1.0", + "ajv-errors": "^1.0.0", + "ajv-keywords": "^3.1.0" + } + }, + "scss-tokenizer": { + "version": "0.2.3", + "resolved": "https://registry.npmmirror.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz", + "integrity": "sha512-dYE8LhncfBUar6POCxMTm0Ln+erjeczqEvCJib5/7XNkdw1FkUGgwMPY360FY0FgPWQxHWCx29Jl3oejyGLM9Q==", + "dev": true, + "requires": { + "js-base64": "^2.1.8", + "source-map": "^0.4.2" + }, + "dependencies": { + "source-map": { + "version": "0.4.4", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.4.4.tgz", + "integrity": "sha512-Y8nIfcb1s/7DcobUz1yOO1GSp7gyL+D9zLHDehT7iRESqGSxjJ448Sg7rvfgsRJCnKLdSl11uGf0s9X80cH0/A==", + "dev": true, + "requires": { + "amdefine": ">=0.0.4" + } + } + } + }, + "select-hose": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/select-hose/-/select-hose-2.0.0.tgz", + "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==", + "dev": true + }, + "selfsigned": { + "version": "1.10.14", + "resolved": "https://registry.npmmirror.com/selfsigned/-/selfsigned-1.10.14.tgz", + "integrity": "sha512-lkjaiAye+wBZDCBsu5BGi0XiLRxeUlsGod5ZP924CRSEoGuZAw/f7y9RKu28rwTfiHVhdavhB0qH0INV6P1lEA==", + "dev": true, + "requires": { + "node-forge": "^0.10.0" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmmirror.com/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "dev": true + }, + "send": { + "version": "0.18.0", + "resolved": "https://registry.npmmirror.com/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dev": true, + "requires": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "dependencies": { + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } + }, + "serialize-javascript": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/serialize-javascript/-/serialize-javascript-4.0.0.tgz", + "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==", + "requires": { + "randombytes": "^2.1.0" + } + }, + "serve-index": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/serve-index/-/serve-index-1.9.1.tgz", + "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==", + "dev": true, + "requires": { + "accepts": "~1.3.4", + "batch": "0.6.1", + "debug": "2.6.9", + "escape-html": "~1.0.3", + "http-errors": "~1.6.2", + "mime-types": "~2.1.17", + "parseurl": "~1.3.2" + }, + "dependencies": { + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true + }, + "http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", + "dev": true, + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true + }, + "setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", + "dev": true + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmmirror.com/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true + } + } + }, + "serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmmirror.com/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dev": true, + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "dev": true + }, + "set-value": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "requires": { + "extend-shallow": "^2.0.1", + "is-extendable": "^0.1.1", + "is-plain-object": "^2.0.3", + "split-string": "^3.0.1" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true + }, + "sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmmirror.com/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "requires": { + "kind-of": "^6.0.2" + } + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "dev": true, + "requires": { + "shebang-regex": "^1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "dev": true + }, + "shell-quote": { + "version": "1.7.3", + "resolved": "https://registry.npmmirror.com/shell-quote/-/shell-quote-1.7.3.tgz", + "integrity": "sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==", + "dev": true + }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmmirror.com/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "simple-git": { + "version": "1.132.0", + "resolved": "https://registry.npmmirror.com/simple-git/-/simple-git-1.132.0.tgz", + "integrity": "sha512-xauHm1YqCTom1sC9eOjfq3/9RKiUA9iPnxBbrY2DdL8l4ADMu0jjM5l5lphQP5YWNqAL2aXC/OeuQ76vHtW5fg==", + "dev": true, + "requires": { + "debug": "^4.0.1" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmmirror.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dev": true, + "requires": { + "is-arrayish": "^0.3.1" + }, + "dependencies": { + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmmirror.com/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "dev": true + } + } + }, + "slash": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/slash/-/slash-2.0.0.tgz", + "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", + "dev": true + }, + "slice-ansi": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/slice-ansi/-/slice-ansi-1.0.0.tgz", + "integrity": "sha512-POqxBK6Lb3q6s047D/XsDVNPnF9Dl8JSaqe9h9lURl0OdNqy/ujDrOiIHtsqXMGbWWTIomRzAMaTyawAU//Reg==", + "dev": true, + "optional": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0" + } + }, + "slide": { + "version": "1.1.6", + "resolved": "https://registry.npmmirror.com/slide/-/slide-1.1.6.tgz", + "integrity": "sha512-NwrtjCg+lZoqhFU8fOwl4ay2ei8PaqCBOUV3/ektPY9trO1yQ1oXEfmHAhKArUVUr/hOHvy5f6AdP17dCM0zMw==", + "dev": true + }, + "smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dev": true + }, + "snapdragon": { + "version": "0.8.2", + "resolved": "https://registry.npmmirror.com/snapdragon/-/snapdragon-0.8.2.tgz", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "requires": { + "base": "^0.11.1", + "debug": "^2.2.0", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "map-cache": "^0.2.2", + "source-map": "^0.5.6", + "source-map-resolve": "^0.5.0", + "use": "^3.1.0" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "snapdragon-node": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "requires": { + "define-property": "^1.0.0", + "isobject": "^3.0.0", + "snapdragon-util": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "requires": { + "is-descriptor": "^1.0.0" + } + }, + "is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "requires": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + } + } + }, + "snapdragon-util": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/snapdragon-util/-/snapdragon-util-3.0.1.tgz", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "requires": { + "kind-of": "^3.2.0" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "sockjs": { + "version": "0.3.24", + "resolved": "https://registry.npmmirror.com/sockjs/-/sockjs-0.3.24.tgz", + "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", + "dev": true, + "requires": { + "faye-websocket": "^0.11.3", + "uuid": "^8.3.2", + "websocket-driver": "^0.7.4" + }, + "dependencies": { + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmmirror.com/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true + } + } + }, + "sockjs-client": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/sockjs-client/-/sockjs-client-1.6.0.tgz", + "integrity": "sha512-qVHJlyfdHFht3eBFZdKEXKTlb7I4IV41xnVNo8yUKA1UHcPJwgW2SvTq9LhnjjCywSkSK7c/e4nghU0GOoMCRQ==", + "dev": true, + "requires": { + "debug": "^3.2.7", + "eventsource": "^1.1.0", + "faye-websocket": "^0.11.4", + "inherits": "^2.0.4", + "url-parse": "^1.5.10" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } + }, + "socks": { + "version": "2.6.2", + "resolved": "https://registry.npmmirror.com/socks/-/socks-2.6.2.tgz", + "integrity": "sha512-zDZhHhZRY9PxRruRMR7kMhnf3I8hDs4S3f9RecfnGxvcBHQcKcIH/oUcEWffsfl1XxdYlA7nnlGbbTvPz9D8gA==", + "dev": true, + "requires": { + "ip": "^1.1.5", + "smart-buffer": "^4.2.0" + } + }, + "socks-proxy-agent": { + "version": "6.2.0", + "resolved": "https://registry.npmmirror.com/socks-proxy-agent/-/socks-proxy-agent-6.2.0.tgz", + "integrity": "sha512-wWqJhjb32Q6GsrUqzuFkukxb/zzide5quXYcMVpIjxalDBBYy2nqKCFQ/9+Ie4dvOYSQdOk3hUlZSdzZOd3zMQ==", + "dev": true, + "requires": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "sort-keys": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/sort-keys/-/sort-keys-1.1.2.tgz", + "integrity": "sha512-vzn8aSqKgytVik0iwdBEi+zevbTYZogewTUM6dtpmGwEcdzbub/TX4bCzRhebDCRC3QzXgJsLRKB2V/Oof7HXg==", + "dev": true, + "requires": { + "is-plain-obj": "^1.0.0" + } + }, + "source-list-map": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/source-list-map/-/source-list-map-2.0.1.tgz", + "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==" + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==" + }, + "source-map-resolve": { + "version": "0.5.3", + "resolved": "https://registry.npmmirror.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "requires": { + "atob": "^2.1.2", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmmirror.com/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + } + } + }, + "source-map-url": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/source-map-url/-/source-map-url-0.4.1.tgz", + "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==" + }, + "spdx-correct": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/spdx-correct/-/spdx-correct-3.1.1.tgz", + "integrity": "sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==", + "dev": true, + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==", + "dev": true + }, + "spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "dev": true, + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.11", + "resolved": "https://registry.npmmirror.com/spdx-license-ids/-/spdx-license-ids-3.0.11.tgz", + "integrity": "sha512-Ctl2BrFiM0X3MANYgj3CkygxhRmr9mi6xhejbdO960nF6EDJApTYpn0BQnDKlnNBULKiCN1n3w9EBkHK8ZWg+g==", + "dev": true + }, + "spdy": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/spdy/-/spdy-4.0.2.tgz", + "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", + "dev": true, + "requires": { + "debug": "^4.1.0", + "handle-thing": "^2.0.0", + "http-deceiver": "^1.2.7", + "select-hose": "^2.0.0", + "spdy-transport": "^3.0.0" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "spdy-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/spdy-transport/-/spdy-transport-3.0.0.tgz", + "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", + "dev": true, + "requires": { + "debug": "^4.1.0", + "detect-node": "^2.0.4", + "hpack.js": "^2.1.6", + "obuf": "^1.1.2", + "readable-stream": "^3.0.6", + "wbuf": "^1.7.3" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "speed-measure-webpack-plugin": { + "version": "1.5.0", + "resolved": "https://registry.npmmirror.com/speed-measure-webpack-plugin/-/speed-measure-webpack-plugin-1.5.0.tgz", + "integrity": "sha512-Re0wX5CtM6gW7bZA64ONOfEPEhwbiSF/vz6e2GvadjuaPrQcHTQdRGsD8+BE7iUOysXH8tIenkPCQBEcspXsNg==", + "dev": true, + "requires": { + "chalk": "^4.1.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "dev": true, + "requires": { + "through": "2" + } + }, + "split-on-first": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/split-on-first/-/split-on-first-1.1.0.tgz", + "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==", + "dev": true + }, + "split-string": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/split-string/-/split-string-3.1.0.tgz", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "requires": { + "extend-shallow": "^3.0.0" + } + }, + "split2": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "dev": true, + "requires": { + "readable-stream": "^3.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, + "sshpk": { + "version": "1.17.0", + "resolved": "https://registry.npmmirror.com/sshpk/-/sshpk-1.17.0.tgz", + "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==", + "dev": true, + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "ssri": { + "version": "6.0.2", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-6.0.2.tgz", + "integrity": "sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==", + "requires": { + "figgy-pudding": "^3.5.1" + } + }, + "stable": { + "version": "0.1.8", + "resolved": "https://registry.npmmirror.com/stable/-/stable-0.1.8.tgz", + "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==" + }, + "stackframe": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/stackframe/-/stackframe-1.2.1.tgz", + "integrity": "sha512-h88QkzREN/hy8eRdyNhhsO7RSJ5oyTqxxmmn0dzBIMUclZsjpfmrsg81vp8mjjAs2vAZ72nyWxRUwSwmh0e4xg==", + "dev": true + }, + "staged-git-files": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/staged-git-files/-/staged-git-files-1.1.2.tgz", + "integrity": "sha512-0Eyrk6uXW6tg9PYkhi/V/J4zHp33aNyi2hOCmhFLqLTIhbgqWn5jlSzI+IU0VqrZq6+DbHcabQl/WP6P3BG0QA==", + "dev": true + }, + "static-extend": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/static-extend/-/static-extend-0.1.2.tgz", + "integrity": "sha512-72E9+uLc27Mt718pMHt9VMNiAL4LMsmDbBva8mxWUCkT07fSzEGMYUCk0XWY6lp0j6RBAG4cJ3mWuZv2OE3s0g==", + "requires": { + "define-property": "^0.2.5", + "object-copy": "^0.1.0" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "requires": { + "is-descriptor": "^0.1.0" + } + } + } + }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true + }, + "stdout-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/stdout-stream/-/stdout-stream-1.4.1.tgz", + "integrity": "sha512-j4emi03KXqJWcIeF8eIXkjMFN1Cmb8gUlDYGeBALLPo5qdyTfA9bOtl8m33lRoC+vFMkP3gl0WsDr6+gzxbbTA==", + "dev": true, + "requires": { + "readable-stream": "^2.0.1" + } + }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g==", + "dev": true + }, + "stream-browserify": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/stream-browserify/-/stream-browserify-2.0.2.tgz", + "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==", + "requires": { + "inherits": "~2.0.1", + "readable-stream": "^2.0.2" + } + }, + "stream-each": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/stream-each/-/stream-each-1.2.3.tgz", + "integrity": "sha512-vlMC2f8I2u/bZGqkdfLQW/13Zihpej/7PmSiMQsbYddxuTsJp8vRe2x2FvVExZg7FaOds43ROAuFJwPR4MTZLw==", + "requires": { + "end-of-stream": "^1.1.0", + "stream-shift": "^1.0.0" + } + }, + "stream-http": { + "version": "2.8.3", + "resolved": "https://registry.npmmirror.com/stream-http/-/stream-http-2.8.3.tgz", + "integrity": "sha512-+TSkfINHDo4J+ZobQLWiMouQYB+UVYFttRA94FpEzzJ7ZdqcL4uUUQ7WkdkI4DSozGmgBUE/a47L+38PenXhUw==", + "requires": { + "builtin-status-codes": "^3.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.3.6", + "to-arraybuffer": "^1.0.0", + "xtend": "^4.0.0" + } + }, + "stream-shift": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + }, + "strict-uri-encode": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", + "integrity": "sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ==", + "dev": true + }, + "string-argv": { + "version": "0.0.2", + "resolved": "https://registry.npmmirror.com/string-argv/-/string-argv-0.0.2.tgz", + "integrity": "sha512-p6/Mqq0utTQWUeGMi/m0uBtlLZEwXSY3+mXzeRRqw7fz5ezUb28Wr0R99NlfbWaMmL/jCyT9be4jpn7Yz8IO8w==", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", + "dev": true + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "string.prototype.padend": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/string.prototype.padend/-/string.prototype.padend-3.1.3.tgz", + "integrity": "sha512-jNIIeokznm8SD/TZISQsZKYu7RJyheFNt84DUPrh482GC8RVp2MKqm2O5oBRdGxbDQoXrhhWtPIWQOiy20svUg==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1" + } + }, + "string.prototype.padstart": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/string.prototype.padstart/-/string.prototype.padstart-3.1.3.tgz", + "integrity": "sha512-NZydyOMtYxpTjGqp0VN5PYUF/tsU15yDMZnUdj16qRUIUiMJkHHSDElYyQFrMu+/WloTpA7MQSiADhBicDfaoA==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1" + } + }, + "string.prototype.trimend": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", + "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "string.prototype.trimstart": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", + "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "stringify-object": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/stringify-object/-/stringify-object-3.3.0.tgz", + "integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==", + "dev": true, + "requires": { + "get-own-enumerable-property-symbols": "^3.0.0", + "is-obj": "^1.0.1", + "is-regexp": "^1.0.0" + }, + "dependencies": { + "is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==", + "dev": true + } + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "dev": true + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==", + "dev": true + }, + "strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true + }, + "strip-indent": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/strip-indent/-/strip-indent-2.0.0.tgz", + "integrity": "sha512-RsSNPLpq6YUL7QYy44RnPVTn/lcVZtb48Uof3X5JLbF4zD/Gs7ZFDv2HWol+leoQN2mT86LAzSshGfkTlSOpsA==", + "dev": true + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, + "optional": true + }, + "strong-log-transformer": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/strong-log-transformer/-/strong-log-transformer-2.1.0.tgz", + "integrity": "sha512-B3Hgul+z0L9a236FAUC9iZsL+nVHgoCJnqCbN588DjYxvGXaXaaFbfmQ/JhvKjZwsOukuR72XbHv71Qkug0HxA==", + "dev": true, + "requires": { + "duplexer": "^0.1.1", + "minimist": "^1.2.0", + "through": "^2.3.4" + } + }, + "stylehacks": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/stylehacks/-/stylehacks-4.0.3.tgz", + "integrity": "sha512-7GlLk9JwlElY4Y6a/rmbH2MhVlTyVmiJd1PfTCqFaIBEGMYNsrO/v3SeGTdhBThLg4Z+NbOk/qFMwCa+J+3p/g==", + "dev": true, + "requires": { + "browserslist": "^4.0.0", + "postcss": "^7.0.0", + "postcss-selector-parser": "^3.0.0" + }, + "dependencies": { + "postcss-selector-parser": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/postcss-selector-parser/-/postcss-selector-parser-3.1.2.tgz", + "integrity": "sha512-h7fJ/5uWuRVyOtkO45pnt1Ih40CEleeyCHzipqAZO2e5H20g25Y48uYnFUiShvY4rZWNJ/Bib/KVPmanaCtOhA==", + "dev": true, + "requires": { + "dot-prop": "^5.2.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + } + } + } + }, + "subarg": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/subarg/-/subarg-1.0.0.tgz", + "integrity": "sha512-RIrIdRY0X1xojthNcVtgT9sjpOGagEUKpZdgBUi054OEPFo282yg+zE+t1Rj3+RqKq2xStL7uUHhY+AjbC4BXg==", + "dev": true, + "requires": { + "minimist": "^1.1.0" + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true + }, + "svg-baker": { + "version": "1.7.0", + "resolved": "https://registry.npmmirror.com/svg-baker/-/svg-baker-1.7.0.tgz", + "integrity": "sha512-nibslMbkXOIkqKVrfcncwha45f97fGuAOn1G99YwnwTj8kF9YiM6XexPcUso97NxOm6GsP0SIvYVIosBis1xLg==", + "dev": true, + "requires": { + "bluebird": "^3.5.0", + "clone": "^2.1.1", + "he": "^1.1.1", + "image-size": "^0.5.1", + "loader-utils": "^1.1.0", + "merge-options": "1.0.1", + "micromatch": "3.1.0", + "postcss": "^5.2.17", + "postcss-prefix-selector": "^1.6.0", + "posthtml-rename-id": "^1.0", + "posthtml-svg-mode": "^1.0.3", + "query-string": "^4.3.2", + "traverse": "^0.6.6" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "dependencies": { + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true + } + } + }, + "clone": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "dev": true + }, + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "has-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-1.0.0.tgz", + "integrity": "sha512-DyYHfIYwAJmjAjSSPKANxI8bFY9YtFrgkAfinBojQ8YJTOuOuav64tMUJv584SES4xl74PmuaevIyaLESHdTAA==", + "dev": true + }, + "is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "requires": { + "kind-of": "^6.0.0" + }, + "dependencies": { + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + } + } + }, + "is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "requires": { + "kind-of": "^6.0.0" + }, + "dependencies": { + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + } + } + }, + "is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "dependencies": { + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + } + } + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + }, + "micromatch": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/micromatch/-/micromatch-3.1.0.tgz", + "integrity": "sha512-3StSelAE+hnRvMs8IdVW7Uhk8CVed5tp+kLLGlBP6WiRAXS21GPGu/Nat4WNPXj2Eoc24B02SaeoyozPMfj0/g==", + "dev": true, + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.2.2", + "define-property": "^1.0.0", + "extend-shallow": "^2.0.1", + "extglob": "^2.0.2", + "fragment-cache": "^0.2.1", + "kind-of": "^5.0.2", + "nanomatch": "^1.2.1", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + } + }, + "postcss": { + "version": "5.2.18", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-5.2.18.tgz", + "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==", + "dev": true, + "requires": { + "chalk": "^1.1.3", + "js-base64": "^2.1.9", + "source-map": "^0.5.6", + "supports-color": "^3.2.3" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "3.2.3", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-3.2.3.tgz", + "integrity": "sha512-Jds2VIYDrlp5ui7t8abHN2bjAu4LV/q4N2KivFPpGH0lrka0BMq/33AmECUXlKPcHigkNaqfXRENFju+rlcy+A==", + "dev": true, + "requires": { + "has-flag": "^1.0.0" + } + } + } + }, + "svg-baker-runtime": { + "version": "1.4.7", + "resolved": "https://registry.npmmirror.com/svg-baker-runtime/-/svg-baker-runtime-1.4.7.tgz", + "integrity": "sha512-Zorfwwj5+lWjk/oxwSMsRdS2sPQQdTmmsvaSpzU+i9ZWi3zugHLt6VckWfnswphQP0LmOel3nggpF5nETbt6xw==", + "dev": true, + "requires": { + "deepmerge": "1.3.2", + "mitt": "1.1.2", + "svg-baker": "^1.7.0" + }, + "dependencies": { + "deepmerge": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/deepmerge/-/deepmerge-1.3.2.tgz", + "integrity": "sha512-qjMjTrk+RKv/sp4RPDpV5CnKhxjFI9p+GkLBOls5A8EEElldYWCWA9zceAkmfd0xIo2aU1nxiaLFoiya2sb6Cg==", + "dev": true + } + } + }, + "svg-sprite-loader": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/svg-sprite-loader/-/svg-sprite-loader-5.0.0.tgz", + "integrity": "sha512-/hedkRC2IS0E+kFIb+OUCfqQlbVx72/WEEeRGw2uPsNgOOgJEONXzjfSm+CJ3pB9gOHytlBmPMS8ijMCp5s2Eg==", + "dev": true, + "requires": { + "bluebird": "^3.5.0", + "deepmerge": "1.3.2", + "domready": "1.0.8", + "escape-string-regexp": "1.0.5", + "html-webpack-plugin": "^3.2.0", + "loader-utils": "^1.1.0", + "svg-baker": "^1.5.0", + "svg-baker-runtime": "^1.4.7", + "url-slug": "2.0.0" + }, + "dependencies": { + "deepmerge": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/deepmerge/-/deepmerge-1.3.2.tgz", + "integrity": "sha512-qjMjTrk+RKv/sp4RPDpV5CnKhxjFI9p+GkLBOls5A8EEElldYWCWA9zceAkmfd0xIo2aU1nxiaLFoiya2sb6Cg==", + "dev": true + } + } + }, + "svg-tags": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/svg-tags/-/svg-tags-1.0.0.tgz", + "integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==", + "dev": true + }, + "svgo": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/svgo/-/svgo-1.3.0.tgz", + "integrity": "sha512-MLfUA6O+qauLDbym+mMZgtXCGRfIxyQoeH6IKVcFslyODEe/ElJNwr0FohQ3xG4C6HK6bk3KYPPXwHVJk3V5NQ==", + "requires": { + "chalk": "^2.4.1", + "coa": "^2.0.2", + "css-select": "^2.0.0", + "css-select-base-adapter": "^0.1.1", + "css-tree": "1.0.0-alpha.33", + "csso": "^3.5.1", + "js-yaml": "^3.13.1", + "mkdirp": "~0.5.1", + "object.values": "^1.1.0", + "sax": "~1.2.4", + "stable": "^0.1.8", + "unquote": "~1.1.1", + "util.promisify": "~1.0.0" + } + }, + "symbol-observable": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/symbol-observable/-/symbol-observable-1.2.0.tgz", + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==", + "dev": true + }, + "synchronous-promise": { + "version": "2.0.15", + "resolved": "https://registry.npmmirror.com/synchronous-promise/-/synchronous-promise-2.0.15.tgz", + "integrity": "sha512-k8uzYIkIVwmT+TcglpdN50pS2y1BDcUnBPK9iJeGu0Pl1lOI8pD6wtzgw91Pjpe+RxtTncw32tLxs/R0yNL2Mg==", + "dev": true + }, + "table": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/table/-/table-4.0.2.tgz", + "integrity": "sha512-UUkEAPdSGxtRpiV9ozJ5cMTtYiqz7Ni1OGqLXRCynrvzdtR1p+cfOWe2RJLwvUG8hNanaSRjecIqwOjqeatDsA==", + "dev": true, + "optional": true, + "requires": { + "ajv": "^5.2.3", + "ajv-keywords": "^2.1.0", + "chalk": "^2.1.0", + "lodash": "^4.17.4", + "slice-ansi": "1.0.0", + "string-width": "^2.1.1" + }, + "dependencies": { + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmmirror.com/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha512-Ajr4IcMXq/2QmMkEmSvxqfLN5zGmJ92gHXAeOXq1OekoH2rfDNsgdDoL2f7QaRCy7G/E6TpxBVdRuNraMztGHw==", + "dev": true, + "optional": true, + "requires": { + "co": "^4.6.0", + "fast-deep-equal": "^1.0.0", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.3.0" + } + }, + "ajv-keywords": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ajv-keywords/-/ajv-keywords-2.1.1.tgz", + "integrity": "sha512-ZFztHzVRdGLAzJmpUT9LNFLe1YiVOEylcaNpEutM26PVTCtOD919IMfD01CgbRouB42Dd9atjx1HseC15DgOZA==", + "dev": true, + "optional": true + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha512-fueX787WZKCV0Is4/T2cyAdM4+x1S3MXXOAhavE1ys/W42SHAPacLTQhucja22QBYrfGw50M2sRiXPtTGv9Ymw==", + "dev": true, + "optional": true + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmmirror.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha512-4JD/Ivzg7PoW8NzdrBSr3UFwC9mHgvI7Z6z3QGBsSHgKaRTUDmyZAAKJo2UbG1kUVfS9WS8bi36N49U1xw43DA==", + "dev": true, + "optional": true + } + } + }, + "tapable": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/tapable/-/tapable-1.1.3.tgz", + "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==" + }, + "tar": { + "version": "4.4.19", + "resolved": "https://registry.npmmirror.com/tar/-/tar-4.4.19.tgz", + "integrity": "sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==", + "dev": true, + "requires": { + "chownr": "^1.1.4", + "fs-minipass": "^1.2.7", + "minipass": "^2.9.0", + "minizlib": "^1.3.3", + "mkdirp": "^0.5.5", + "safe-buffer": "^5.2.1", + "yallist": "^3.1.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + } + } + }, + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "temp-dir": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/temp-dir/-/temp-dir-1.0.0.tgz", + "integrity": "sha512-xZFXEGbG7SNC3itwBzI3RYjq/cEhBkx2hJuKGIUOcEULmkQExXiHat2z/qkISYsuR+IKumhEfKKbV5qXmhICFQ==", + "dev": true + }, + "temp-write": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/temp-write/-/temp-write-4.0.0.tgz", + "integrity": "sha512-HIeWmj77uOOHb0QX7siN3OtwV3CTntquin6TNVg6SHOqCP3hYKmox90eeFOGaY1MqJ9WYDDjkyZrW6qS5AWpbw==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.15", + "is-stream": "^2.0.0", + "make-dir": "^3.0.0", + "temp-dir": "^1.0.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "requires": { + "semver": "^6.0.0" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "terser": { + "version": "4.8.0", + "resolved": "https://registry.npmmirror.com/terser/-/terser-4.8.0.tgz", + "integrity": "sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==", + "requires": { + "commander": "^2.20.0", + "source-map": "~0.6.1", + "source-map-support": "~0.5.12" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + } + } + }, + "terser-webpack-plugin": { + "version": "1.4.5", + "resolved": "https://registry.npmmirror.com/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz", + "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==", + "requires": { + "cacache": "^12.0.2", + "find-cache-dir": "^2.1.0", + "is-wsl": "^1.1.0", + "schema-utils": "^1.0.0", + "serialize-javascript": "^4.0.0", + "source-map": "^0.6.1", + "terser": "^4.1.2", + "webpack-sources": "^1.4.0", + "worker-farm": "^1.7.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + } + } + }, + "text-extensions": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/text-extensions/-/text-extensions-1.9.0.tgz", + "integrity": "sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ==", + "dev": true + }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true + }, + "thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "requires": { + "any-promise": "^1.0.0" + } + }, + "thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "requires": { + "thenify": ">= 3.1.0 < 4" + } + }, + "thread-loader": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/thread-loader/-/thread-loader-2.1.3.tgz", + "integrity": "sha512-wNrVKH2Lcf8ZrWxDF/khdlLlsTMczdcwPA9VEK4c2exlEPynYWxi9op3nPTo5lAnDIkE0rQEB3VBP+4Zncc9Hg==", + "dev": true, + "requires": { + "loader-runner": "^2.3.1", + "loader-utils": "^1.1.0", + "neo-async": "^2.6.0" + } + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmmirror.com/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "through2": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "requires": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "thunky": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/thunky/-/thunky-1.1.0.tgz", + "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==", + "dev": true + }, + "timers-browserify": { + "version": "2.0.12", + "resolved": "https://registry.npmmirror.com/timers-browserify/-/timers-browserify-2.0.12.tgz", + "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==", + "requires": { + "setimmediate": "^1.0.4" + } + }, + "timsort": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/timsort/-/timsort-0.3.0.tgz", + "integrity": "sha512-qsdtZH+vMoCARQtyod4imc2nIJwg9Cc7lPRrw9CzF8ZKR0khdr8+2nX80PBhET3tcyTtJDxAffGh2rXH4tyU8A==", + "dev": true + }, + "tinycolor2": { + "version": "1.4.2", + "resolved": "https://registry.npmmirror.com/tinycolor2/-/tinycolor2-1.4.2.tgz", + "integrity": "sha512-vJhccZPs965sV/L2sU4oRQVAos0pQXwsvTLkWYdqJ+a8Q5kPFzJTuOFwy7UniPli44NKQGAglksjvOcpo95aZA==" + }, + "tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmmirror.com/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dev": true, + "requires": { + "os-tmpdir": "~1.0.2" + } + }, + "to-arraybuffer": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", + "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==" + }, + "to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "dev": true + }, + "to-object-path": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/to-object-path/-/to-object-path-0.3.0.tgz", + "integrity": "sha512-9mWHdnGRuh3onocaHzukyvCZhzvr6tiflAy/JRFXcJX0TjgfWA9pk9t8CMbzmBE4Jfw58pXbkngtBtqYxzNEyg==", + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "to-regex": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/to-regex/-/to-regex-3.0.2.tgz", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "requires": { + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "regex-not": "^1.0.2", + "safe-regex": "^1.1.0" + } + }, + "to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==", + "requires": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + } + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true + }, + "toposort": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/toposort/-/toposort-1.0.7.tgz", + "integrity": "sha512-FclLrw8b9bMWf4QlCJuHBEVhSRsqDj6u3nIjAzPeJvgl//1hBlffdlk0MALceL14+koWEdU4ofRAXofbODxQzg==", + "dev": true + }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmmirror.com/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + }, + "tr46": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "dev": true, + "requires": { + "punycode": "^2.1.1" + } + }, + "traverse": { + "version": "0.6.6", + "resolved": "https://registry.npmmirror.com/traverse/-/traverse-0.6.6.tgz", + "integrity": "sha512-kdf4JKs8lbARxWdp7RKdNzoJBhGUcIalSYibuGyHJbmk40pOysQ0+QPvlkCOICOivDWU2IJo2rkrxyTK2AH4fw==", + "dev": true + }, + "trim-newlines": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/trim-newlines/-/trim-newlines-3.0.1.tgz", + "integrity": "sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==", + "dev": true + }, + "true-case-path": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/true-case-path/-/true-case-path-1.0.3.tgz", + "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", + "dev": true, + "requires": { + "glob": "^7.1.2" + } + }, + "tryer": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/tryer/-/tryer-1.0.1.tgz", + "integrity": "sha512-c3zayb8/kWWpycWYg87P71E1S1ZL6b6IJxfb5fvsUgsf0S2MVGaDhDXXjDMpdCpfWXqptc+4mXwmiy1ypXqRAA==", + "dev": true + }, + "tsconfig-paths": { + "version": "3.14.1", + "resolved": "https://registry.npmmirror.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz", + "integrity": "sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==", + "dev": true, + "requires": { + "@types/json5": "^0.0.29", + "json5": "^1.0.1", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + } + }, + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmmirror.com/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" + }, + "tty-browserify": { + "version": "0.0.0", + "resolved": "https://registry.npmmirror.com/tty-browserify/-/tty-browserify-0.0.0.tgz", + "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==" + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmmirror.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmmirror.com/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "dev": true + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmmirror.com/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2" + } + }, + "type-fest": { + "version": "0.6.0", + "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.6.0.tgz", + "integrity": "sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==", + "dev": true + }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmmirror.com/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, + "typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmmirror.com/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" + }, + "typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmmirror.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, + "requires": { + "is-typedarray": "^1.0.0" + } + }, + "uglify-js": { + "version": "3.4.10", + "resolved": "https://registry.npmmirror.com/uglify-js/-/uglify-js-3.4.10.tgz", + "integrity": "sha512-Y2VsbPVs0FIshJztycsO2SfPk7/KAF/T72qzv9u5EpQ4kB2hQoHlhNQTsNyy6ul7lQtqJN/AoWeS23OzEiEFxw==", + "dev": true, + "requires": { + "commander": "~2.19.0", + "source-map": "~0.6.1" + }, + "dependencies": { + "commander": { + "version": "2.19.0", + "resolved": "https://registry.npmmirror.com/commander/-/commander-2.19.0.tgz", + "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==", + "dev": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "uid-number": { + "version": "0.0.6", + "resolved": "https://registry.npmmirror.com/uid-number/-/uid-number-0.0.6.tgz", + "integrity": "sha512-c461FXIljswCuscZn67xq9PpszkPT6RjheWFQTgCyabJrTUozElanb0YEqv2UGgk247YpcJkFBuSGNvBlpXM9w==", + "dev": true + }, + "umask": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/umask/-/umask-1.1.0.tgz", + "integrity": "sha512-lE/rxOhmiScJu9L6RTNVgB/zZbF+vGC0/p6D3xnkAePI2o0sMyFG966iR5Ki50OI/0mNi2yaRnxfLsPmEZF/JA==", + "dev": true + }, + "unbox-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", + "requires": { + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", + "which-boxed-primitive": "^1.0.2" + } + }, + "unicode-canonical-property-names-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", + "integrity": "sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==", + "dev": true + }, + "unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "dev": true, + "requires": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + } + }, + "unicode-match-property-value-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.0.0.tgz", + "integrity": "sha512-7Yhkc0Ye+t4PNYzOGKedDhXbYIBe1XEQYQxOPyhcXNMJ0WCABqqj6ckydd6pWRZTHV4GuCPKdBAUiMc60tsKVw==", + "dev": true + }, + "unicode-property-aliases-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.0.0.tgz", + "integrity": "sha512-5Zfuy9q/DFr4tfO7ZPeVXb1aPoeQSdeFMLpYuFebehDAhbuevLs5yxSZmIFN1tP5F9Wl4IpJrYojg85/zgyZHQ==", + "dev": true + }, + "unidecode": { + "version": "0.1.8", + "resolved": "https://registry.npmmirror.com/unidecode/-/unidecode-0.1.8.tgz", + "integrity": "sha512-SdoZNxCWpN2tXTCrGkPF/0rL2HEq+i2gwRG1ReBvx8/0yTzC3enHfugOf8A9JBShVwwrRIkLX0YcDUGbzjbVCA==", + "dev": true + }, + "union-value": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "requires": { + "arr-union": "^3.1.0", + "get-value": "^2.0.6", + "is-extendable": "^0.1.1", + "set-value": "^2.0.1" + } + }, + "uniq": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/uniq/-/uniq-1.0.1.tgz", + "integrity": "sha512-Gw+zz50YNKPDKXs+9d+aKAjVwpjNwqzvNpLigIruT4HA9lMZNdMqs9x07kKHB/L9WRzqp4+DlTU5s4wG2esdoA==", + "dev": true + }, + "uniqs": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/uniqs/-/uniqs-2.0.0.tgz", + "integrity": "sha512-mZdDpf3vBV5Efh29kMw5tXoup/buMgxLzOt/XKFKcVmi+15ManNQWr6HfZ2aiZTYlYixbdNJ0KFmIZIv52tHSQ==", + "dev": true + }, + "unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "requires": { + "unique-slug": "^2.0.0" + } + }, + "unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "requires": { + "imurmurhash": "^0.1.4" + } + }, + "universal-user-agent": { + "version": "6.0.0", + "resolved": "https://registry.npmmirror.com/universal-user-agent/-/universal-user-agent-6.0.0.tgz", + "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==", + "dev": true + }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true + }, + "unquote": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/unquote/-/unquote-1.1.1.tgz", + "integrity": "sha512-vRCqFv6UhXpWxZPyGDh/F3ZpNv8/qo7w6iufLpQg9aKnQ71qM4B5KiI7Mia9COcjEhrO9LueHpMYjYzsWH3OIg==" + }, + "unset-value": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/unset-value/-/unset-value-1.0.0.tgz", + "integrity": "sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==", + "requires": { + "has-value": "^0.3.1", + "isobject": "^3.0.0" + }, + "dependencies": { + "has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmmirror.com/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q==", + "requires": { + "get-value": "^2.0.3", + "has-values": "^0.1.4", + "isobject": "^2.0.0" + }, + "dependencies": { + "isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "requires": { + "isarray": "1.0.0" + } + } + } + }, + "has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==" + } + } + }, + "upath": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/upath/-/upath-1.2.0.tgz", + "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==" + }, + "upper-case": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/upper-case/-/upper-case-1.1.3.tgz", + "integrity": "sha512-WRbjgmYzgXkCV7zNVpy5YgrHgbBv126rMALQQMrmzOVC4GM2waQ9x7xtm8VU+1yF2kWyPzI9zbZ48n4vSxwfSA==", + "dev": true + }, + "uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmmirror.com/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "requires": { + "punycode": "^2.1.0" + } + }, + "urix": { + "version": "0.1.0", + "resolved": "https://registry.npmmirror.com/urix/-/urix-0.1.0.tgz", + "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==" + }, + "url": { + "version": "0.11.0", + "resolved": "https://registry.npmmirror.com/url/-/url-0.11.0.tgz", + "integrity": "sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" + } + } + }, + "url-loader": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/url-loader/-/url-loader-1.1.2.tgz", + "integrity": "sha512-dXHkKmw8FhPqu8asTc1puBfe3TehOCo2+RmOOev5suNCIYBcT626kxiWg1NBVkwc4rO8BGa7gP70W7VXuqHrjg==", + "dev": true, + "requires": { + "loader-utils": "^1.1.0", + "mime": "^2.0.3", + "schema-utils": "^1.0.0" + } + }, + "url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmmirror.com/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "url-slug": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/url-slug/-/url-slug-2.0.0.tgz", + "integrity": "sha512-aiNmSsVgrjCiJ2+KWPferjT46YFKoE8i0YX04BlMVDue022Xwhg/zYlnZ6V9/mP3p8Wj7LEp0myiTkC/p6sxew==", + "dev": true, + "requires": { + "unidecode": "0.1.8" + } + }, + "use": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/use/-/use-3.1.1.tgz", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==" + }, + "util": { + "version": "0.11.1", + "resolved": "https://registry.npmmirror.com/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "requires": { + "inherits": "2.0.3" + }, + "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + } + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "util-promisify": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/util-promisify/-/util-promisify-2.1.0.tgz", + "integrity": "sha512-K+5eQPYs14b3+E+hmE2J6gCZ4JmMl9DbYS6BeP2CHq6WMuNxErxf5B/n0fz85L8zUuoO6rIzNNmIQDu/j+1OcA==", + "dev": true, + "requires": { + "object.getownpropertydescriptors": "^2.0.3" + } + }, + "util.promisify": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/util.promisify/-/util.promisify-1.0.1.tgz", + "integrity": "sha512-g9JpC/3He3bm38zsLupWryXHoEcS22YHthuPQSJdMy6KNrzIRzWqcsHzD/WUnqe45whVou4VIsPew37DoXWNrA==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.2", + "has-symbols": "^1.0.1", + "object.getownpropertydescriptors": "^2.1.0" + } + }, + "utila": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/utila/-/utila-0.4.0.tgz", + "integrity": "sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==", + "dev": true + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true + }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmmirror.com/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "dev": true + }, + "v-click-outside-x": { + "version": "4.1.3", + "resolved": "https://registry.npmmirror.com/v-click-outside-x/-/v-click-outside-x-4.1.3.tgz", + "integrity": "sha512-qK4wwuDHK406fGBSJ4DbioPb6LQpRkWqk8i1TZhnVwObU+W4Ra6H7Cn+VLy/dTNTH/sgZJzk9YMYnB/s5RJ0Hg==" + }, + "v8-compile-cache": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", + "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", + "dev": true + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "validate-npm-package-name": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/validate-npm-package-name/-/validate-npm-package-name-3.0.0.tgz", + "integrity": "sha512-M6w37eVCMMouJ9V/sdPGnC5H4uDr73/+xdq0FBLO3TFFX1+7wiUY6Es328NN+y43tmY+doUdN9g9J21vqB7iLw==", + "dev": true, + "requires": { + "builtins": "^1.0.3" + } + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true + }, + "vendors": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/vendors/-/vendors-1.0.4.tgz", + "integrity": "sha512-/juG65kTL4Cy2su4P8HjtkTxk6VmJDiOPBufWniqQ6wknac6jNiXS9vU+hO3wgusiyqWlzTbVHi0dyJqRONg3w==", + "dev": true + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmmirror.com/verror/-/verror-1.10.0.tgz", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", + "dev": true, + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + }, + "dependencies": { + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==", + "dev": true + } + } + }, + "vm-browserify": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/vm-browserify/-/vm-browserify-1.1.2.tgz", + "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" + }, + "vue": { + "version": "2.6.12", + "resolved": "https://registry.npmmirror.com/vue/-/vue-2.6.12.tgz", + "integrity": "sha512-uhmLFETqPPNyuLLbsKz6ioJ4q7AZHzD8ZVFNATNyICSZouqP2Sz0rotWQC8UNBF6VGSCs5abnKJoStA6JbCbfg==" + }, + "vue-cli-plugin-mockjs": { + "version": "0.1.3", + "resolved": "https://registry.npmmirror.com/vue-cli-plugin-mockjs/-/vue-cli-plugin-mockjs-0.1.3.tgz", + "integrity": "sha512-BK7EaGhrLYYMOAPuhLxZjpBGJOLtpCFfVT2FDhDqBZIF+DimmAG/06P+gFwMDSQb8j8ROirLLBgLDjqIXJj56A==", + "dev": true, + "requires": { + "express": "^4.16.4", + "mockjs": "^1.0.1-beta3" + } + }, + "vue-eslint-parser": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/vue-eslint-parser/-/vue-eslint-parser-2.0.3.tgz", + "integrity": "sha512-ZezcU71Owm84xVF6gfurBQUGg8WQ+WZGxgDEQu1IHFBZNx7BFZg3L1yHxrCBNNwbwFtE1GuvfJKMtb6Xuwc/Bw==", + "dev": true, + "optional": true, + "requires": { + "debug": "^3.1.0", + "eslint-scope": "^3.7.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^3.5.2", + "esquery": "^1.0.0", + "lodash": "^4.17.4" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "optional": true, + "requires": { + "ms": "^2.1.1" + } + }, + "eslint-scope": { + "version": "3.7.3", + "resolved": "https://registry.npmmirror.com/eslint-scope/-/eslint-scope-3.7.3.tgz", + "integrity": "sha512-W+B0SvF4gamyCTmUc+uITPY0989iXVfKvhwtmJocTaYoc/3khEHmEmvfY/Gn9HA9VV75jrQECsHizkNw1b68FA==", + "dev": true, + "optional": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "optional": true + } + } + }, + "vue-hot-reload-api": { + "version": "2.3.4", + "resolved": "https://registry.npmmirror.com/vue-hot-reload-api/-/vue-hot-reload-api-2.3.4.tgz", + "integrity": "sha512-BXq3jwIagosjgNVae6tkHzzIk6a8MHFtzAdwhnV5VlvPTFxDCvIttgSiHWjdGoTJvXtmRu5HacExfdarRcFhog==", + "dev": true + }, + "vue-i18n": { + "version": "8.22.1", + "resolved": "https://registry.npmmirror.com/vue-i18n/-/vue-i18n-8.22.1.tgz", + "integrity": "sha512-JNgiEJ5a8YPfk5y2lKyfOAGLmkpAVfhaUi+T4wGpSppRYZ3XSyawSDDketY5KV2CsAiBLAGEIO6jO+0l2hQubg==" + }, + "vue-loader": { + "version": "15.9.8", + "resolved": "https://registry.npmmirror.com/vue-loader/-/vue-loader-15.9.8.tgz", + "integrity": "sha512-GwSkxPrihfLR69/dSV3+5CdMQ0D+jXg8Ma1S4nQXKJAznYFX14vHdc/NetQc34Dw+rBbIJyP7JOuVb9Fhprvog==", + "dev": true, + "requires": { + "@vue/component-compiler-utils": "^3.1.0", + "hash-sum": "^1.0.2", + "loader-utils": "^1.1.0", + "vue-hot-reload-api": "^2.3.0", + "vue-style-loader": "^4.1.0" + } + }, + "vue-router": { + "version": "3.4.8", + "resolved": "https://registry.npmmirror.com/vue-router/-/vue-router-3.4.8.tgz", + "integrity": "sha512-3BsR84AqarcmweXjItxw3jwQsiYNssYg090yi4rlzTnCJxmHtkyCvhNz9Z7qRSOkmiV485KkUCReTp5AjNY4wg==" + }, + "vue-style-loader": { + "version": "4.1.3", + "resolved": "https://registry.npmmirror.com/vue-style-loader/-/vue-style-loader-4.1.3.tgz", + "integrity": "sha512-sFuh0xfbtpRlKfm39ss/ikqs9AbKCoXZBpHeVZ8Tx650o0k0q/YCM7FRvigtxpACezfq6af+a7JeqVTWvncqDg==", + "dev": true, + "requires": { + "hash-sum": "^1.0.2", + "loader-utils": "^1.0.2" + } + }, + "vue-template-compiler": { + "version": "2.6.12", + "resolved": "https://registry.npmmirror.com/vue-template-compiler/-/vue-template-compiler-2.6.12.tgz", + "integrity": "sha512-OzzZ52zS41YUbkCBfdXShQTe69j1gQDZ9HIX8miuC9C3rBCk9wIRjLiZZLrmX9V+Ftq/YEyv1JaVr5Y/hNtByg==", + "dev": true, + "requires": { + "de-indent": "^1.0.2", + "he": "^1.1.0" + } + }, + "vue-template-es2015-compiler": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/vue-template-es2015-compiler/-/vue-template-es2015-compiler-1.9.1.tgz", + "integrity": "sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==", + "dev": true + }, + "vuescroll": { + "version": "4.16.1", + "resolved": "https://registry.npmmirror.com/vuescroll/-/vuescroll-4.16.1.tgz", + "integrity": "sha512-7fRsG2Yw5Z07LUz/IIu9barpmYiN9q+ZTC+CrVamvCbmsxyhz8mU1OuYFbfORysaUskioNMxTGDo+HOzeDfSyQ==" + }, + "watchpack": { + "version": "1.7.5", + "resolved": "https://registry.npmmirror.com/watchpack/-/watchpack-1.7.5.tgz", + "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==", + "requires": { + "chokidar": "^3.4.1", + "graceful-fs": "^4.1.2", + "neo-async": "^2.5.0", + "watchpack-chokidar2": "^2.0.1" + } + }, + "watchpack-chokidar2": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz", + "integrity": "sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==", + "optional": true, + "requires": { + "chokidar": "^2.1.8" + }, + "dependencies": { + "anymatch": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-2.0.0.tgz", + "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==", + "optional": true, + "requires": { + "micromatch": "^3.1.4", + "normalize-path": "^2.1.1" + }, + "dependencies": { + "normalize-path": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-2.1.1.tgz", + "integrity": "sha512-3pKJwH184Xo/lnH6oyP1q2pMd7HcypqqmRs91/6/i2CGtWwIKGCkOOMTm/zXbgTEWHw1uNpNi/igc3ePOYHb6w==", + "optional": true, + "requires": { + "remove-trailing-separator": "^1.0.1" + } + } + } + }, + "binary-extensions": { + "version": "1.13.1", + "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-1.13.1.tgz", + "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", + "optional": true + }, + "chokidar": { + "version": "2.1.8", + "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-2.1.8.tgz", + "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "optional": true, + "requires": { + "anymatch": "^2.0.0", + "async-each": "^1.0.1", + "braces": "^2.3.2", + "fsevents": "^1.2.7", + "glob-parent": "^3.1.0", + "inherits": "^2.0.3", + "is-binary-path": "^1.0.0", + "is-glob": "^4.0.0", + "normalize-path": "^3.0.0", + "path-is-absolute": "^1.0.0", + "readdirp": "^2.2.1", + "upath": "^1.1.1" + } + }, + "fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } + }, + "glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==", + "optional": true, + "requires": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + }, + "dependencies": { + "is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==", + "optional": true, + "requires": { + "is-extglob": "^2.1.0" + } + } + } + }, + "is-binary-path": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-1.0.1.tgz", + "integrity": "sha512-9fRVlXc0uCxEDj1nQzaWONSpbTfx0FmJfzHF7pwlI8DkWGoHBBea4Pg5Ky0ojwwxQmnSifgbKkI06Qv0Ljgj+Q==", + "optional": true, + "requires": { + "binary-extensions": "^1.0.0" + } + }, + "readdirp": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-2.2.1.tgz", + "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "optional": true, + "requires": { + "graceful-fs": "^4.1.11", + "micromatch": "^3.1.10", + "readable-stream": "^2.0.2" + } + } + } + }, + "wbuf": { + "version": "1.7.3", + "resolved": "https://registry.npmmirror.com/wbuf/-/wbuf-1.7.3.tgz", + "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", + "dev": true, + "requires": { + "minimalistic-assert": "^1.0.0" + } + }, + "wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "dev": true, + "requires": { + "defaults": "^1.0.3" + } + }, + "webidl-conversions": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz", + "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "dev": true + }, + "webpack": { + "version": "4.46.0", + "resolved": "https://registry.npmmirror.com/webpack/-/webpack-4.46.0.tgz", + "integrity": "sha512-6jJuJjg8znb/xRItk7bkT0+Q7AHCYjjFnvKIWQPkNIOyRqoCGvkOs0ipeQzrqz4l5FtN5ZI/ukEHroeX/o1/5Q==", + "requires": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-module-context": "1.9.0", + "@webassemblyjs/wasm-edit": "1.9.0", + "@webassemblyjs/wasm-parser": "1.9.0", + "acorn": "^6.4.1", + "ajv": "^6.10.2", + "ajv-keywords": "^3.4.1", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^4.5.0", + "eslint-scope": "^4.0.3", + "json-parse-better-errors": "^1.0.2", + "loader-runner": "^2.4.0", + "loader-utils": "^1.2.3", + "memory-fs": "^0.4.1", + "micromatch": "^3.1.10", + "mkdirp": "^0.5.3", + "neo-async": "^2.6.1", + "node-libs-browser": "^2.2.1", + "schema-utils": "^1.0.0", + "tapable": "^1.1.3", + "terser-webpack-plugin": "^1.4.3", + "watchpack": "^1.7.4", + "webpack-sources": "^1.4.1" + } + }, + "webpack-bundle-analyzer": { + "version": "3.9.0", + "resolved": "https://registry.npmmirror.com/webpack-bundle-analyzer/-/webpack-bundle-analyzer-3.9.0.tgz", + "integrity": "sha512-Ob8amZfCm3rMB1ScjQVlbYYUEJyEjdEtQ92jqiFUYt5VkEeO2v5UMbv49P/gnmCZm3A6yaFQzCBvpZqN4MUsdA==", + "dev": true, + "requires": { + "acorn": "^7.1.1", + "acorn-walk": "^7.1.1", + "bfj": "^6.1.1", + "chalk": "^2.4.1", + "commander": "^2.18.0", + "ejs": "^2.6.1", + "express": "^4.16.3", + "filesize": "^3.6.1", + "gzip-size": "^5.0.0", + "lodash": "^4.17.19", + "mkdirp": "^0.5.1", + "opener": "^1.5.1", + "ws": "^6.0.0" + }, + "dependencies": { + "acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmmirror.com/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true + }, + "acorn-walk": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/acorn-walk/-/acorn-walk-7.2.0.tgz", + "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", + "dev": true + } + } + }, + "webpack-chain": { + "version": "4.12.1", + "resolved": "https://registry.npmmirror.com/webpack-chain/-/webpack-chain-4.12.1.tgz", + "integrity": "sha512-BCfKo2YkDe2ByqkEWe1Rw+zko4LsyS75LVr29C6xIrxAg9JHJ4pl8kaIZ396SUSNp6b4815dRZPSTAS8LlURRQ==", + "dev": true, + "requires": { + "deepmerge": "^1.5.2", + "javascript-stringify": "^1.6.0" + }, + "dependencies": { + "deepmerge": { + "version": "1.5.2", + "resolved": "https://registry.npmmirror.com/deepmerge/-/deepmerge-1.5.2.tgz", + "integrity": "sha512-95k0GDqvBjZavkuvzx/YqVLv/6YYa17fz6ILMSf7neqQITCPbnfEnQvEgMPNjH4kgobe7+WIL0yJEHku+H3qtQ==", + "dev": true + } + } + }, + "webpack-dev-middleware": { + "version": "3.7.3", + "resolved": "https://registry.npmmirror.com/webpack-dev-middleware/-/webpack-dev-middleware-3.7.3.tgz", + "integrity": "sha512-djelc/zGiz9nZj/U7PTBi2ViorGJXEWo/3ltkPbDyxCXhhEXkW0ce99falaok4TPj+AsxLiXJR0EBOb0zh9fKQ==", + "dev": true, + "requires": { + "memory-fs": "^0.4.1", + "mime": "^2.4.4", + "mkdirp": "^0.5.1", + "range-parser": "^1.2.1", + "webpack-log": "^2.0.0" + } + }, + "webpack-dev-server": { + "version": "3.11.3", + "resolved": "https://registry.npmmirror.com/webpack-dev-server/-/webpack-dev-server-3.11.3.tgz", + "integrity": "sha512-3x31rjbEQWKMNzacUZRE6wXvUFuGpH7vr0lIEbYpMAG9BOxi0928QU1BBswOAP3kg3H1O4hiS+sq4YyAn6ANnA==", + "dev": true, + "requires": { + "ansi-html-community": "0.0.8", + "bonjour": "^3.5.0", + "chokidar": "^2.1.8", + "compression": "^1.7.4", + "connect-history-api-fallback": "^1.6.0", + "debug": "^4.1.1", + "del": "^4.1.1", + "express": "^4.17.1", + "html-entities": "^1.3.1", + "http-proxy-middleware": "0.19.1", + "import-local": "^2.0.0", + "internal-ip": "^4.3.0", + "ip": "^1.1.5", + "is-absolute-url": "^3.0.3", + "killable": "^1.0.1", + "loglevel": "^1.6.8", + "opn": "^5.5.0", + "p-retry": "^3.0.1", + "portfinder": "^1.0.26", + "schema-utils": "^1.0.0", + "selfsigned": "^1.10.8", + "semver": "^6.3.0", + "serve-index": "^1.9.1", + "sockjs": "^0.3.21", + "sockjs-client": "^1.5.0", + "spdy": "^4.0.2", + "strip-ansi": "^3.0.1", + "supports-color": "^6.1.0", + "url": "^0.11.0", + "webpack-dev-middleware": "^3.7.2", + "webpack-log": "^2.0.0", + "ws": "^6.2.1", + "yargs": "^13.3.2" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "anymatch": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-2.0.0.tgz", + "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==", + "dev": true, + "requires": { + "micromatch": "^3.1.4", + "normalize-path": "^2.1.1" + }, + "dependencies": { + "normalize-path": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-2.1.1.tgz", + "integrity": "sha512-3pKJwH184Xo/lnH6oyP1q2pMd7HcypqqmRs91/6/i2CGtWwIKGCkOOMTm/zXbgTEWHw1uNpNi/igc3ePOYHb6w==", + "dev": true, + "requires": { + "remove-trailing-separator": "^1.0.1" + } + } + } + }, + "binary-extensions": { + "version": "1.13.1", + "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-1.13.1.tgz", + "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", + "dev": true + }, + "chokidar": { + "version": "2.1.8", + "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-2.1.8.tgz", + "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "dev": true, + "requires": { + "anymatch": "^2.0.0", + "async-each": "^1.0.1", + "braces": "^2.3.2", + "fsevents": "^1.2.7", + "glob-parent": "^3.1.0", + "inherits": "^2.0.3", + "is-binary-path": "^1.0.0", + "is-glob": "^4.0.0", + "normalize-path": "^3.0.0", + "path-is-absolute": "^1.0.0", + "readdirp": "^2.2.1", + "upath": "^1.1.1" + } + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "dev": true, + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } + }, + "glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==", + "dev": true, + "requires": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + }, + "dependencies": { + "is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==", + "dev": true, + "requires": { + "is-extglob": "^2.1.0" + } + } + } + }, + "is-absolute-url": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/is-absolute-url/-/is-absolute-url-3.0.3.tgz", + "integrity": "sha512-opmNIX7uFnS96NtPmhWQgQx6/NYFgsUXYMllcfzwWKUMwfo8kku1TvE6hkNcH+Q1ts5cMVrsY7j0bxXQDciu9Q==", + "dev": true + }, + "is-binary-path": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-1.0.1.tgz", + "integrity": "sha512-9fRVlXc0uCxEDj1nQzaWONSpbTfx0FmJfzHF7pwlI8DkWGoHBBea4Pg5Ky0ojwwxQmnSifgbKkI06Qv0Ljgj+Q==", + "dev": true, + "requires": { + "binary-extensions": "^1.0.0" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "readdirp": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-2.2.1.tgz", + "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11", + "micromatch": "^3.1.10", + "readable-stream": "^2.0.2" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "dev": true + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "yargs": { + "version": "13.3.2", + "resolved": "https://registry.npmmirror.com/yargs/-/yargs-13.3.2.tgz", + "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "dev": true, + "requires": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.2" + } + }, + "yargs-parser": { + "version": "13.1.2", + "resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + } + } + }, + "webpack-log": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/webpack-log/-/webpack-log-2.0.0.tgz", + "integrity": "sha512-cX8G2vR/85UYG59FgkoMamwHUIkSSlV3bBMRsbxVXVUk2j6NleCKjQ/WE9eYg9WY4w25O9w8wKP4rzNZFmUcUg==", + "dev": true, + "requires": { + "ansi-colors": "^3.0.0", + "uuid": "^3.3.2" + } + }, + "webpack-merge": { + "version": "4.2.2", + "resolved": "https://registry.npmmirror.com/webpack-merge/-/webpack-merge-4.2.2.tgz", + "integrity": "sha512-TUE1UGoTX2Cd42j3krGYqObZbOD+xF7u28WB7tfUordytSjbWTIjK/8V0amkBfTYN4/pB/GIDlJZZ657BGG19g==", + "dev": true, + "requires": { + "lodash": "^4.17.15" + } + }, + "webpack-sources": { + "version": "1.4.3", + "resolved": "https://registry.npmmirror.com/webpack-sources/-/webpack-sources-1.4.3.tgz", + "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "requires": { + "source-list-map": "^2.0.0", + "source-map": "~0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + } + } + }, + "webpack-virtual-modules": { + "version": "0.3.2", + "resolved": "https://registry.npmmirror.com/webpack-virtual-modules/-/webpack-virtual-modules-0.3.2.tgz", + "integrity": "sha512-RXQXioY6MhzM4CNQwmBwKXYgBs6ulaiQ8bkNQEl2J6Z+V+s7lgl/wGvaI/I0dLnYKB8cKsxQc17QOAVIphPLDw==", + "dev": true, + "requires": { + "debug": "^3.0.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } + }, + "websocket-driver": { + "version": "0.7.4", + "resolved": "https://registry.npmmirror.com/websocket-driver/-/websocket-driver-0.7.4.tgz", + "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", + "dev": true, + "requires": { + "http-parser-js": ">=0.5.1", + "safe-buffer": ">=5.1.0", + "websocket-extensions": ">=0.1.1" + } + }, + "websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", + "dev": true + }, + "whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmmirror.com/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dev": true, + "requires": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmmirror.com/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "requires": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", + "dev": true + }, + "wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmmirror.com/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "dev": true, + "requires": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true + }, + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "dev": true + }, + "worker-farm": { + "version": "1.7.0", + "resolved": "https://registry.npmmirror.com/worker-farm/-/worker-farm-1.7.0.tgz", + "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", + "requires": { + "errno": "~0.1.7" + } + }, + "worker-loader": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/worker-loader/-/worker-loader-2.0.0.tgz", + "integrity": "sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw==", + "requires": { + "loader-utils": "^1.0.0", + "schema-utils": "^0.4.0" + }, + "dependencies": { + "schema-utils": { + "version": "0.4.7", + "resolved": "https://registry.npmmirror.com/schema-utils/-/schema-utils-0.4.7.tgz", + "integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==", + "requires": { + "ajv": "^6.1.0", + "ajv-keywords": "^3.1.0" + } + } + } + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "write": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/write/-/write-0.2.1.tgz", + "integrity": "sha512-CJ17OoULEKXpA5pef3qLj5AxTJ6mSt7g84he2WIskKwqFO4T97d5V7Tadl0DYDk7qyUOQD5WlUlOMChaYrhxeA==", + "dev": true, + "optional": true, + "requires": { + "mkdirp": "^0.5.1" + } + }, + "write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmmirror.com/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + }, + "write-json-file": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/write-json-file/-/write-json-file-4.3.0.tgz", + "integrity": "sha512-PxiShnxf0IlnQuMYOPPhPkhExoCQuTUNPOa/2JWCYTmBquU9njyyDuwRKN26IZBlp4yn1nt+Agh2HOOBl+55HQ==", + "dev": true, + "requires": { + "detect-indent": "^6.0.0", + "graceful-fs": "^4.1.15", + "is-plain-obj": "^2.0.0", + "make-dir": "^3.0.0", + "sort-keys": "^4.0.0", + "write-file-atomic": "^3.0.0" + }, + "dependencies": { + "detect-indent": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/detect-indent/-/detect-indent-6.1.0.tgz", + "integrity": "sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==", + "dev": true + }, + "is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "dev": true + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "requires": { + "semver": "^6.0.0" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "sort-keys": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/sort-keys/-/sort-keys-4.2.0.tgz", + "integrity": "sha512-aUYIEU/UviqPgc8mHR6IW1EGxkAXpeRETYcrzg8cLAvUPZcpAlleSXHV2mY7G12GphSH6Gzv+4MMVSSkbdteHg==", + "dev": true, + "requires": { + "is-plain-obj": "^2.0.0" + } + }, + "write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, + "requires": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + } + } + }, + "write-pkg": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/write-pkg/-/write-pkg-4.0.0.tgz", + "integrity": "sha512-v2UQ+50TNf2rNHJ8NyWttfm/EJUBWMJcx6ZTYZr6Qp52uuegWw/lBkCtCbnYZEmPRNL61m+u67dAmGxo+HTULA==", + "dev": true, + "requires": { + "sort-keys": "^2.0.0", + "type-fest": "^0.4.1", + "write-json-file": "^3.2.0" + }, + "dependencies": { + "sort-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/sort-keys/-/sort-keys-2.0.0.tgz", + "integrity": "sha512-/dPCrG1s3ePpWm6yBbxZq5Be1dXGLyLn9Z791chDC3NFrpkVbWGzkBwPN1knaciexFXgRJ7hzdnwZ4stHSDmjg==", + "dev": true, + "requires": { + "is-plain-obj": "^1.0.0" + } + }, + "type-fest": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.4.1.tgz", + "integrity": "sha512-IwzA/LSfD2vC1/YDYMv/zHP4rDF1usCwllsDpbolT3D4fUepIO7f9K70jjmUewU/LmGUKJcwcVtDCpnKk4BPMw==", + "dev": true + }, + "write-json-file": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/write-json-file/-/write-json-file-3.2.0.tgz", + "integrity": "sha512-3xZqT7Byc2uORAatYiP3DHUUAVEkNOswEWNs9H5KXiicRTvzYzYqKjYc4G7p+8pltvAw641lVByKVtMpf+4sYQ==", + "dev": true, + "requires": { + "detect-indent": "^5.0.0", + "graceful-fs": "^4.1.15", + "make-dir": "^2.1.0", + "pify": "^4.0.1", + "sort-keys": "^2.0.0", + "write-file-atomic": "^2.4.2" + } + } + } + }, + "ws": { + "version": "6.2.2", + "resolved": "https://registry.npmmirror.com/ws/-/ws-6.2.2.tgz", + "integrity": "sha512-zmhltoSR8u1cnDsD43TX59mzoMZsLKqUweyYBAIvTngR3shc0W6aOZylZmq/7hqyVxPdi+5Ud2QInblgyE72fw==", + "dev": true, + "requires": { + "async-limiter": "~1.0.0" + } + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, + "y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + }, + "yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmmirror.com/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "dev": true + }, + "yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmmirror.com/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmmirror.com/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmmirror.com/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true + } + } + }, + "yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "dev": true + }, + "yorkie": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/yorkie/-/yorkie-2.0.0.tgz", + "integrity": "sha512-jcKpkthap6x63MB4TxwCyuIGkV0oYP/YRyuQU5UO0Yz/E/ZAu+653/uov+phdmO54n6BcvFRyyt0RRrWdN2mpw==", + "dev": true, + "requires": { + "execa": "^0.8.0", + "is-ci": "^1.0.10", + "normalize-path": "^1.0.0", + "strip-indent": "^2.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "execa": { + "version": "0.8.0", + "resolved": "https://registry.npmmirror.com/execa/-/execa-0.8.0.tgz", + "integrity": "sha512-zDWS+Rb1E8BlqqhALSt9kUhss8Qq4nN3iof3gsOdyINksElaPyNBtKUMTR62qhvgVWR0CqCX7sdnKe4MnUbFEA==", + "dev": true, + "requires": { + "cross-spawn": "^5.0.1", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==", + "dev": true + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "normalize-path": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-1.0.0.tgz", + "integrity": "sha512-7WyT0w8jhpDStXRq5836AMmihQwq2nrUVQrgjvUo/p/NZf9uy/MeJ246lBJVmWuYXMlJuG9BNZHF0hWjfTbQUA==", + "dev": true + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", + "dev": true + } + } + }, + "yup": { + "version": "0.27.0", + "resolved": "https://registry.npmmirror.com/yup/-/yup-0.27.0.tgz", + "integrity": "sha512-v1yFnE4+u9za42gG/b/081E7uNW9mUj3qtkmelLbW5YPROZzSH/KUUyJu9Wt8vxFJcT9otL/eZopS0YK1L5yPQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.0.0", + "fn-name": "~2.0.1", + "lodash": "^4.17.11", + "property-expr": "^1.5.0", + "synchronous-promise": "^2.0.6", + "toposort": "^2.0.2" + }, + "dependencies": { + "toposort": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/toposort/-/toposort-2.0.2.tgz", + "integrity": "sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==", + "dev": true + } + } + }, + "zip-stream": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/zip-stream/-/zip-stream-2.1.3.tgz", + "integrity": "sha512-EkXc2JGcKhO5N5aZ7TmuNo45budRaFGHOmz24wtJR7znbNqDPmdZtUauKX6et8KAVseAMBOyWJqEpXcHTBsh7Q==", + "dev": true, + "requires": { + "archiver-utils": "^2.1.0", + "compress-commons": "^2.1.1", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "zrender": { + "version": "4.3.2", + "resolved": "https://registry.npmmirror.com/zrender/-/zrender-4.3.2.tgz", + "integrity": "sha512-bIusJLS8c4DkIcdiK+s13HiQ/zjQQVgpNohtd8d94Y2DnJqgM1yjh/jpDb8DoL6hd7r8Awagw8e3qK/oLaWr3g==" + } + } +} diff --git a/web/package.json b/web/package.json new file mode 100644 index 000000000..7d905cc8d --- /dev/null +++ b/web/package.json @@ -0,0 +1,73 @@ +{ + "name": "dataspherestudio", + "version": "1.1.4", + "private": true, + "scripts": { + "serve": "npm run postinstall && cd packages/dss && npm run serve", + "build": "npm run postinstall && cd packages/dss && npm run build", + "lint": "vue-cli-service lint --no-fix", + "fix": "vue-cli-service lint --fix", + "precommit": "lint-staged", + "postinstall": "patch-package" + }, + "husky": { + "hooks": { + "postcommit": "git update-index --again", + "pre-commit": "lint-staged" + } + }, + "lint-staged": { + "packages/**/*.{js,vue}": [ + "vue-cli-service lint --no-fix", + "git add" + ] + }, + "dependencies": { + "axios": "0.21.1", + "babel-polyfill": "6.26.0", + "butterfly-dag": "4.1.23", + "core-js": "2.6.11", + "echarts": "^4.1.0", + "iview": "3.5.4", + "jsencrypt": "^3.2.1", + "lodash": "4.17.20", + "md5": "2.3.0", + "moment": "2.29.4", + "monaco-editor": "0.19.3", + "qs": "6.9.4", + "svgo": "1.3.0", + "vue": "2.6.12", + "vue-i18n": "8.22.1", + "vue-router": "3.4.8", + "vuescroll": "4.16.1", + "webpack": "^4.46.0", + "worker-loader": "2.0.0" + }, + "devDependencies": { + "@kazupon/vue-i18n-loader": "0.4.1", + "@vue/cli-plugin-babel": "3.12.1", + "@vue/cli-plugin-eslint": "3.12.1", + "@vue/cli-service": "3.12.1", + "@vue/eslint-config-standard": "4.0.0", + "archiver": "3.1.1", + "babel-eslint": "10.1.0", + "copy-webpack-plugin": "4.6.0", + "csp-html-webpack-plugin": "4.0.0", + "eslint": "6.8.0", + "eslint-plugin-vue": "6.2.2", + "filemanager-webpack-plugin": "2.0.5", + "husky": "1.3.1", + "lerna": "^4.0.0", + "less-loader": "6.1.0", + "lint-staged": "8.2.1", + "monaco-editor-webpack-plugin": "1.8.2", + "node-sass": "4.14.1", + "sass-loader": "7.3.1", + "speed-measure-webpack-plugin": "1.5.0", + "svg-sprite-loader": "5.0.0", + "patch-package": "6.2.2", + "vue-cli-plugin-mockjs": "0.1.3", + "vue-template-compiler": "2.6.12", + "webpack-virtual-modules": "0.3.2" + } +} diff --git a/web/packages/apiServices/assets/images/dssLogo.png b/web/packages/apiServices/assets/images/dssLogo.png new file mode 100644 index 000000000..2619ce7d1 Binary files /dev/null and b/web/packages/apiServices/assets/images/dssLogo.png differ diff --git a/web/packages/apiServices/i18n/en.json b/web/packages/apiServices/i18n/en.json new file mode 100644 index 000000000..05923fe8c --- /dev/null +++ b/web/packages/apiServices/i18n/en.json @@ -0,0 +1,262 @@ +{ + "message": { + "apiServices": { + "viewAll": "View All", + "viewValid": "View Valid", + "deleted": "Deleted", + "delete": "Delete", + "enterDetele": "confirm Detele", + "enterDeteleName": "Are you sure to delete {name}?", + "publishing": "Publishing", + "confirm": "Confirm", + "modifyRemarks": "Modify Remarks", + "save": "Save", + "open": "Open", + "title": "Data Service", + "back": "Back", + "tableDisplay": "Table", + "cardDisplay": "Card", + "display": "Display", + "action": "Aciton", + "kill": { + "title": "Execution Notice", + "desc": "Stopped executing this script" + }, + "date": { + "label": " Start Date", + "placeholder": "Please choose the start date" + }, + "query": { + "apiName": "Data Service Title", + "tag": "Tag", + "status": "Service Status", + "commiter": "Commiter", + "buttonText": "Query", + "clearButtonText": "Clear", + "stop": "Stop", + "comment": "Comment", + "more": "More" + }, + "formTitle": { + "dataApiManage": "Data API Manage" + }, + "shortcuts": { + "week": "Recent Week", + "month": "Recent Month", + "threeMonths": "Recent Three Months" + }, + "placeholder": { + "enterName": "Please enter the name", + "apiName": "Enter API Name", + "tag": "Select Tag", + "scope": "Select Scope", + "description": "Enter Description", + "visibleRole": "Select Visible Role", + "emter": "Enter", + "startDate": "Please choose the start date", + "dateAndTime": "Select Date And Time", + "inputSubmitter": "Input Submitter" + }, + "label": { + "apiName": "API Name:", + "tag": "Tag:", + "name": "Name", + "parameter": "Parameter Info:", + "scope": "Scope:", + "description": "Description:", + "visibleRole": "Visible Role:", + "test": "Test:", + "applicant": "Applicant:", + "status": "Status:", + "find": "Find", + "path": "API Path:", + "submitter": "Submitter", + "scriptsPath": "Scripts Path:" + }, + "buttonText": { + "enter": "Enter", + "maintained": "Being maintained", + "refresh": "Refresh Status" + }, + "tip": { + "success": "Success", + "cannotBeEmpty": "Cannot Be Empty", + "failed": "Failed", + "disableConfirmTitle": "Disable Confirm", + "disableConfirm": "Are you sure you want to disable the {name} API ?", + "enableConfirmTitle": "Enable Confirm", + "enableConfirm": "Are you sure you want to enable the {name} API ?" + }, + "apiTable": { + "user": "User", + "duration": "Duration", + "reason": "Reason", + "ip_whitelist": "Ip Whitelist", + "apply_time": "Apply Time", + "caller": "Caller", + "access": "Access", + "version": "Version", + "scriptPath": "Script Path", + "creator": "Creator", + "publishDateStr": "Publish DateStr", + "updateDateStr": "Update Date", + "apiName": "Data Service Name", + "apiPath": "API Path", + "status": "Status", + "type": "Type", + "describe": "Describe", + "calledCount": "Call count", + "responsiblePerson": "Responsible Person", + "belongTo": "Belong To", + "tag": "Tag", + "operation": { + "title": "Operation", + "test": "Test", + "copy": "Copy", + "disable": "Disable", + "enable": "Enable", + "manager": "Manager", + "update": "Update", + "view": "View" + }, + "reconfirm": { + "title": "Confirm", + "disable": "Confirm disable this API?", + "enable": "Confirm enable this API?" + } + }, + "view": { + "scriptPath": "Script Path:", + "scriptContent": "Script Content:" + }, + "enable": "Running", + "disable": "Stop", + "normal": "Normal", + "expired": "Expired", + "all": "All", + "updateApiModal": { + "modalTitle": "Update API", + "selectApi": "API Name", + "apiPath": "API Path", + "apiVersion": "API Version", + "apiVersionUpgrade": "New Version", + "paramConfirm": "Please confirm param" + }, + "paramTable": { + "paramName": "Param Name", + "paramType": "Param Type", + "require": { + "title": "Require", + "yes": "Yes", + "no": "No", + "hide": "Hide" + }, + "defaultValue": "Default Value", + "displayName": "Display Name", + "describe": "Describe" + }, + "rule": { + "nameRule": "Please input api name!", + "pathRule": "Please input api path!", + "pathRegRule": "Api path illegal!", + "requestTypeRule": "Please select request type!", + "protocolRule": "Please select protocol!", + "contentLengLimit": "Length limit 255", + "pathRepeat": "Api path exists", + "nameRepeat": "Api name exists" + }, + "visible": { + "workspace": "Workspace", + "personal": "Personal", + "public": "Public" + }, + "apiInfo": { + "title": "API Info", + "authorize": "Token Authorize", + "set": "Set", + "tag": "Tag", + "metrics": "API Metrics", + "monthCall": "Call Total This Month", + "weekCall": "Call Total This Week", + "dayCall": "Call Total Today", + "averageTime": "Average Call Time", + "callSuccess": "Success Percent", + "topQps": "TOP QPS", + "use": "API Call Info", + "selectCallTime": "Select Call Time", + "version": "Version", + "callStatus": "Call Info", + "callUser": "Caller", + "callApp": "CallApp", + "callIp": "Call IP", + "callTime": "Call Time", + "callVersion": "Call Version", + "operation": "Operation", + "query": { + "clearButtonText": "Clear", + "buttonText": "Query" + } + }, + "apiSetForm": { + "flowLimitation": "Flow Limitation", + "defaultSetting": "Default Setting", + "advancedSetting": "Advanced Setting", + "form": { + "MaxConcurrentNumber": "Max Concurrent Number:", + "MaxSimultaneousQuery": "Max Simultaneous Query:", + "MaxTimeQuery": "Max Time Query:", + "defaultVersion": "Default Version:", + "defaultConfigurationParameter": "Default Configuration Parameter:", + "defaultVariable": "Default Variable:", + "approvalNo": "Approval No:" + } + }, + "apiTestInfo": { + "title": "API Test", + "testApi": "Test API", + "apiVersion": "Version", + "search": "Search", + "params": "Screening criteria", + "moreParams": "More filters", + "paramsMore": "More filters", + "selectParams": "已选择过滤条件", + "allSelect": "全选", + "allSelectParams": "可选择参数列表", + "paramTable": { + "paramName": "Param Name", + "paramType": "Param Type", + "description": "Param Description", + "require": { + "title": "Require", + "yes": "YES", + "no": "NO" + }, + "defaultValue": "Test Value" + }, + "testResult": "Test Result" + }, + "apiVersionInfo": { + "title": "API Version", + "params": "params", + "versionInfoTable": { + "version": "Version", + "status": "Status", + "source": "Source", + "creator": "Publisher", + "publishDate": "Publish Time", + "updateDate": "Update Time", + "operation": "Operation", + "query": { + "title": "Query", + "clear": "Clear", + "publishDate": "Select Publish Time" + } + } + }, + "notice": { + "publishSuccess": "Publish Success" + }, + "tagPlaceholder": "tag,click enter add" + } + } +} diff --git a/web/packages/apiServices/i18n/zh.json b/web/packages/apiServices/i18n/zh.json new file mode 100644 index 000000000..cd6505c21 --- /dev/null +++ b/web/packages/apiServices/i18n/zh.json @@ -0,0 +1,262 @@ +{ + "message": { + "apiServices": { + "viewAll": "显示全部", + "viewValid": "屏蔽已删除", + "deleted": "已删除", + "delete": "删除", + "enterDetele": "确认删除", + "enterDeteleName": "确认删除{name}服务?", + "publishing": "发布中", + "confirm": "确认", + "modifyRemarks": "修改备注", + "save": "保存", + "open": "打开", + "title": "数据服务", + "back": "返回", + "tableDisplay": "列表展示", + "cardDisplay": "图标展示", + "display": "展示方式", + "action": "操作", + "kill": { + "title": "运行提示", + "desc": "已经停止执行脚本" + }, + "date": { + "label": "起始时间", + "placeholder": "请选择起始日期" + }, + "query": { + "apiName": "数据服务名称", + "tag": "标签", + "status": "服务状态", + "commiter": "提交人", + "buttonText": "查询", + "clearButtonText": "清空", + "stop": "停止", + "comment": "备注", + "more": "更多" + }, + "formTitle": { + "dataApiManage": "数据API管理" + }, + "shortcuts": { + "week": "最近一周", + "month": "最近一个月", + "threeMonths": "最近三个月" + }, + "placeholder": { + "enterName": "请输入名称", + "apiName": "请输入API名称", + "tag": "请选择标签", + "scope": "请选择可见范围", + "description": "请输入描述", + "visibleRole": "请选择可见角色", + "emter": "请输入", + "startDate": "请选择起始日期", + "dateAndTime": "请选择日期和时间", + "inputSubmitter": "输入提交人" + }, + "label": { + "apiName": "API名称:", + "tag": "标签:", + "name": "名称", + "parameter": "参数信息:", + "scope": "可见范围:", + "description": "描述:", + "visibleRole": "可见角色:", + "test": "测试:", + "applicant": "申请人:", + "status": "状态:", + "path": "API路径:", + "submitter": "提交人", + "find": "查询", + "scriptsPath": "脚本路径:" + }, + "buttonText": { + "enter": "进入使用", + "maintained": "禁止使用", + "refresh": "刷新状态" + }, + "tip": { + "success": "保存成功", + "failed": "不通过", + "disableConfirmTitle": "禁用确认", + "enableConfirmTitle": "开启确认", + "disableConfirm": "确认禁用 {name} api吗?", + "enableConfirm": "确认开启 {name} api吗?", + "cannotBeEmpty": "不能为空!" + }, + "apiTable": { + "apiName": "数据服务名称", + "apiPath": "API 路径", + "status": "状态", + "type": "类型", + "describe": "描述", + "calledCount": "调用次数", + "responsiblePerson": "责任人", + "belongTo": "API归属", + "tag": "标签", + "user": "申请用户", + "duration": "生效时长", + "reason": "申请原因", + "ip_whitelist": "白名单IP", + "apply_time": "申请起始时间", + "caller": "调用方", + "access": "限流情况", + "version": "版本号", + "scriptPath": "来源", + "creator": "发布者", + "publishDateStr": "发布时间", + "updateDateStr": "更新时间", + "operation": { + "title": "操作", + "test": "测试", + "copy": "复制", + "disable": "禁用", + "enable": "启用", + "manager": "管理", + "update": "更新", + "view": "查看" + }, + "reconfirm": { + "title": "确认", + "disable": "确定禁用该API吗?", + "enable": "确定启动该API吗?" + } + }, + "view": { + "scriptPath": "脚本路径:", + "scriptContent": "脚本内容:" + }, + "enable": "运行中", + "disable": "已禁用", + "normal": "正常", + "expired": "已过期", + "all": "全部", + "updateApiModal": { + "modalTitle": "数据服务API更新", + "selectApi": "API 名称", + "apiPath": "API 路径", + "apiVersion": "API版本", + "apiVersionUpgrade": "生成新版本", + "paramConfirm": "请确认参数" + }, + "paramTable": { + "paramName": "参数名称", + "paramType": "参数类型", + "require": { + "title": "是否必填", + "yes": "是", + "no": "否", + "hide": "隐藏" + }, + "defaultValue": "默认值", + "displayName": "展示名", + "describe": "描述" + }, + "rule": { + "nameRule": "请填写API名称", + "pathRule": "请填写API路径", + "pathRegRule": "API路径不合法", + "requestTypeRule": "请选择请求方式", + "protocolRule": "请选择协议", + "contentLengthLimit": "长度不能大于255", + "pathRepeat": "API路径已存在", + "nameRepeat": "API名称已存在" + }, + "visible": { + "workspace": "工作空间", + "personal": "个人", + "public": "公开" + }, + "notice": { + "publishSuccess": "发布成功" + }, + "apiVersionInfo": { + "title": "API版本", + "versionInfoTable": { + "version": "版本", + "status": "状态", + "source": "来源", + "creator": "发布者", + "publishDate": "发布时间", + "updateDate": "最后更新时间", + "operation": "操作", + "query": { + "title": "查询", + "clear": "清空", + "publishDate": "请选择发布时间" + } + } + }, + "apiTestInfo": { + "title": "API测试", + "testApi": "测试API", + "apiVersion": "请求版本", + "search": "搜索", + "params": "筛选条件", + "moreParams": "更多筛选条件...", + "paramsMore": "筛选更多", + "selectParams": "已选择过滤条件", + "allSelect": "全选", + "allSelectParams": "可选择参数列表", + "paramTable": { + "paramName": "参数名称", + "paramType": "参数类型", + "description": "参数描述", + "displayName": "展示名", + "require": { + "title": "是否必填", + "yes": "是", + "no": "否" + }, + "defaultValue": "测试值" + }, + "testResult": "测试结果" + }, + "apiSetForm": { + "flowLimitation": "限流设置", + "defaultSetting": "默认设置", + "advancedSetting": "高级设置", + "form": { + "MaxConcurrentNumber": "最大并发数:", + "MaxSimultaneousQuery": "最大同时在查数:", + "MaxTimeQuery": "最大查询时长:", + "defaultVersion": "默认版本:", + "defaultConfigurationParameter": "默认配置参数:", + "defaultVariable": "默认变量:", + "approvalNo": "审批单号:" + } + }, + "apiInfo": { + "title": "API信息", + "set": "设置", + "authorize": "Token授权", + "tag": "标签", + "metrics": "API Metrics", + "monthCall": "本月总调用次数", + "weekCall": "本周总调用次数", + "dayCall": "今日调用次数", + "averageTime": "平均调用时长", + "callSuccess": "调用成功率", + "topQps": "最高QPS", + "use": "API调用情况", + "selectCallTime": "请选择调用时间", + "version": "版本", + "callStatus": "调用情况", + "callUser": "调用者", + "callApp": "调用方", + "callIp": "调用IP", + "callTime": "调用时长", + "callVersion": "调用版本", + "operation": "操作", + "query": { + "clearButtonText": "清空", + "buttonText": "查询" + } + }, + "tagPlaceholder": "标签,按 enter 创建" + } + } +} diff --git a/web/packages/apiServices/module/apiServices/apiCard.vue b/web/packages/apiServices/module/apiServices/apiCard.vue new file mode 100644 index 000000000..0af6174a3 --- /dev/null +++ b/web/packages/apiServices/module/apiServices/apiCard.vue @@ -0,0 +1,190 @@ + + + + diff --git a/web/packages/apiServices/module/apiServices/dropdownCom.vue b/web/packages/apiServices/module/apiServices/dropdownCom.vue new file mode 100644 index 000000000..e75d4725f --- /dev/null +++ b/web/packages/apiServices/module/apiServices/dropdownCom.vue @@ -0,0 +1,35 @@ + + + diff --git a/web/packages/apiServices/module/apiServices/form.vue b/web/packages/apiServices/module/apiServices/form.vue new file mode 100644 index 000000000..979703614 --- /dev/null +++ b/web/packages/apiServices/module/apiServices/form.vue @@ -0,0 +1,180 @@ + + + diff --git a/web/packages/apiServices/module/apiServices/index.js b/web/packages/apiServices/module/apiServices/index.js new file mode 100644 index 000000000..c598d94a9 --- /dev/null +++ b/web/packages/apiServices/module/apiServices/index.js @@ -0,0 +1,10 @@ +export default { + name: 'apiServices', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `${location.protocol}//${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/packages/apiServices/module/apiServices/index.vue b/web/packages/apiServices/module/apiServices/index.vue new file mode 100644 index 000000000..823f471ee --- /dev/null +++ b/web/packages/apiServices/module/apiServices/index.vue @@ -0,0 +1,756 @@ + + + + diff --git a/web/packages/apiServices/module/apiServicesExecute/index.js b/web/packages/apiServices/module/apiServicesExecute/index.js new file mode 100644 index 000000000..1b478dba9 --- /dev/null +++ b/web/packages/apiServices/module/apiServicesExecute/index.js @@ -0,0 +1,11 @@ +export default { + name: 'dataServicesMangement', + events: [], + dispatchs: { + IndexedDB: ['updateResult'], + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `${location.protocol}//${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/packages/apiServices/module/apiServicesExecute/index.vue b/web/packages/apiServices/module/apiServicesExecute/index.vue new file mode 100644 index 000000000..bf3945bee --- /dev/null +++ b/web/packages/apiServices/module/apiServicesExecute/index.vue @@ -0,0 +1,353 @@ + + + + diff --git a/web/packages/apiServices/module/header/index.js b/web/packages/apiServices/module/header/index.js new file mode 100644 index 000000000..4e0a78d62 --- /dev/null +++ b/web/packages/apiServices/module/header/index.js @@ -0,0 +1,5 @@ +export default { + name: 'scriptisHeader', + component: () => import('./index.vue'), + dispatchs: [], +}; \ No newline at end of file diff --git a/web/packages/apiServices/module/header/index.scss b/web/packages/apiServices/module/header/index.scss new file mode 100644 index 000000000..fd86b1b0a --- /dev/null +++ b/web/packages/apiServices/module/header/index.scss @@ -0,0 +1,21 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +@import '@dataspherestudio/shared/common/style/variables.scss'; + +@import '@dataspherestudio/shared/common/style/headerCommon.scss'; + diff --git a/web/packages/apiServices/module/header/index.vue b/web/packages/apiServices/module/header/index.vue new file mode 100644 index 000000000..e8bcbb86c --- /dev/null +++ b/web/packages/apiServices/module/header/index.vue @@ -0,0 +1,101 @@ + + + diff --git a/web/packages/apiServices/module/header/userMenu.vue b/web/packages/apiServices/module/header/userMenu.vue new file mode 100644 index 000000000..38e34ca8b --- /dev/null +++ b/web/packages/apiServices/module/header/userMenu.vue @@ -0,0 +1,109 @@ + + + diff --git a/web/packages/apiServices/module/servicesMangement/component/APIInfo.vue b/web/packages/apiServices/module/servicesMangement/component/APIInfo.vue new file mode 100644 index 000000000..a0ffd19d8 --- /dev/null +++ b/web/packages/apiServices/module/servicesMangement/component/APIInfo.vue @@ -0,0 +1,57 @@ + + + + diff --git a/web/packages/apiServices/module/servicesMangement/component/APIVersions.vue b/web/packages/apiServices/module/servicesMangement/component/APIVersions.vue new file mode 100644 index 000000000..d6ae30a3e --- /dev/null +++ b/web/packages/apiServices/module/servicesMangement/component/APIVersions.vue @@ -0,0 +1,281 @@ + + + + diff --git a/web/packages/apiServices/module/servicesMangement/component/TokenAuthorization.vue b/web/packages/apiServices/module/servicesMangement/component/TokenAuthorization.vue new file mode 100644 index 000000000..d12307ab8 --- /dev/null +++ b/web/packages/apiServices/module/servicesMangement/component/TokenAuthorization.vue @@ -0,0 +1,217 @@ + + + + diff --git a/web/packages/apiServices/module/servicesMangement/index.js b/web/packages/apiServices/module/servicesMangement/index.js new file mode 100644 index 000000000..ccb438d08 --- /dev/null +++ b/web/packages/apiServices/module/servicesMangement/index.js @@ -0,0 +1,10 @@ +export default { + name: 'servicesMangement', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `${location.protocol}//${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/packages/apiServices/module/servicesMangement/index.vue b/web/packages/apiServices/module/servicesMangement/index.vue new file mode 100644 index 000000000..6b3e978dd --- /dev/null +++ b/web/packages/apiServices/module/servicesMangement/index.vue @@ -0,0 +1,378 @@ + + + + + diff --git a/web/packages/apiServices/package.json b/web/packages/apiServices/package.json new file mode 100644 index 000000000..1f636f402 --- /dev/null +++ b/web/packages/apiServices/package.json @@ -0,0 +1,7 @@ +{ + "name": "@dataspherestudio/apiServices", + "version": "1.1.4", + "dependencies": { + "@dataspherestudio/shared": "^1.1.4" + } +} diff --git a/web/packages/apiServices/router.js b/web/packages/apiServices/router.js new file mode 100644 index 000000000..ea60b7038 --- /dev/null +++ b/web/packages/apiServices/router.js @@ -0,0 +1,28 @@ +const routes = [ + { + path: '/apiservices', + name: 'Apiservices', + meta: { + publicPage: true + }, + component: () => import('./view/apiServices/index.vue') + }, + { + path: '/servicesMangement', + name: 'ServicesMangement', + meta: { + publicPage: true + }, + component: () => import('./view/servicesMangement/index.vue') + }, + { + path: '/servicesExecute', + name: 'ServicesExecute', + meta: { + publicPage: true + }, + component: () => import('./view/servicesExecute/index.vue') + }, +] + +export default routes; diff --git a/web/packages/apiServices/view/apiServices/index.vue b/web/packages/apiServices/view/apiServices/index.vue new file mode 100644 index 000000000..3f90d2ad2 --- /dev/null +++ b/web/packages/apiServices/view/apiServices/index.vue @@ -0,0 +1,14 @@ + + + \ No newline at end of file diff --git a/web/packages/apiServices/view/servicesExecute/index.vue b/web/packages/apiServices/view/servicesExecute/index.vue new file mode 100644 index 000000000..b55324d67 --- /dev/null +++ b/web/packages/apiServices/view/servicesExecute/index.vue @@ -0,0 +1,12 @@ + + + \ No newline at end of file diff --git a/web/packages/apiServices/view/servicesMangement/index.vue b/web/packages/apiServices/view/servicesMangement/index.vue new file mode 100644 index 000000000..717da2223 --- /dev/null +++ b/web/packages/apiServices/view/servicesMangement/index.vue @@ -0,0 +1,15 @@ + + + \ No newline at end of file diff --git a/web/packages/dataGovernance/assets/images/dssLogo.png b/web/packages/dataGovernance/assets/images/dssLogo.png new file mode 100644 index 000000000..2619ce7d1 Binary files /dev/null and b/web/packages/dataGovernance/assets/images/dssLogo.png differ diff --git "a/web/packages/dataGovernance/assets/images/\346\220\234\347\264\242\346\241\206\350\243\205\351\245\260\345\233\276.png" "b/web/packages/dataGovernance/assets/images/\346\220\234\347\264\242\346\241\206\350\243\205\351\245\260\345\233\276.png" new file mode 100644 index 000000000..6535e44f8 Binary files /dev/null and "b/web/packages/dataGovernance/assets/images/\346\220\234\347\264\242\346\241\206\350\243\205\351\245\260\345\233\276.png" differ diff --git a/web/packages/dataGovernance/directive/clipboard/clipboard.js b/web/packages/dataGovernance/directive/clipboard/clipboard.js new file mode 100644 index 000000000..dd8ff6709 --- /dev/null +++ b/web/packages/dataGovernance/directive/clipboard/clipboard.js @@ -0,0 +1,49 @@ +const Clipboard = require('clipboard'); + +if (!Clipboard) { + throw new Error('you should npm install `clipboard` --save at first '); +} + +export default { + bind(el, binding) { + if (binding.arg === 'success') { + el._v_clipboard_success = binding.value; + } else if (binding.arg === 'error') { + el._v_clipboard_error = binding.value; + } else { + const clipboard = new Clipboard(el, { + text() { return binding.value; }, + action() { return binding.arg === 'cut' ? 'cut' : 'copy'; } + }); + clipboard.on('success', (e) => { + const callback = el._v_clipboard_success; + callback && callback(e) // eslint-disable-line + }); + clipboard.on('error', (e) => { + const callback = el._v_clipboard_error; + callback && callback(e) // eslint-disable-line + }); + el._v_clipboard = clipboard; + } + }, + update(el, binding) { + if (binding.arg === 'success') { + el._v_clipboard_success = binding.value; + } else if (binding.arg === 'error') { + el._v_clipboard_error = binding.value; + } else { + el._v_clipboard.text = function () { return binding.value; }; + el._v_clipboard.action = function () { return binding.arg === 'cut' ? 'cut' : 'copy'; }; + } + }, + unbind(el, binding) { + if (binding.arg === 'success') { + delete el._v_clipboard_success; + } else if (binding.arg === 'error') { + delete el._v_clipboard_error; + } else { + el._v_clipboard.destroy(); + delete el._v_clipboard; + } + } +}; diff --git a/web/packages/dataGovernance/directive/clipboard/index.js b/web/packages/dataGovernance/directive/clipboard/index.js new file mode 100644 index 000000000..3783c2d4a --- /dev/null +++ b/web/packages/dataGovernance/directive/clipboard/index.js @@ -0,0 +1,13 @@ +import Clipboard from './clipboard'; + +const install = (Vue) => { + Vue.directive('Clipboard', Clipboard); +}; + +if (window.Vue) { + window.clipboard = Clipboard; + Vue.use(install); // eslint-disable-line +} + +Clipboard.install = install; +export default Clipboard; diff --git a/web/packages/dataGovernance/i18n/en.json b/web/packages/dataGovernance/i18n/en.json new file mode 100644 index 000000000..f520c5410 --- /dev/null +++ b/web/packages/dataGovernance/i18n/en.json @@ -0,0 +1,14 @@ +{ + "message": { + "dataGovernance": { + "dataGovernance": "Data Governance", + "dataAssets": "Data Assets", + "dataOverview": "Data Overview", + "dataAssetsIndex": "Data Assets Index", + "overallMeasurement": "Overall measurement", + "menu": "Menu", + "pleaseEnterATableName": "Please enter [database name / table name / field name / table description] to search the data table vaguely", + "search": "Search" + } + } +} diff --git a/web/packages/dataGovernance/i18n/zh.json b/web/packages/dataGovernance/i18n/zh.json new file mode 100644 index 000000000..8debd7f1c --- /dev/null +++ b/web/packages/dataGovernance/i18n/zh.json @@ -0,0 +1,14 @@ +{ + "message": { + "dataGovernance": { + "dataGovernance": "数据治理", + "dataAssets": "数据资产", + "dataOverview": "数据总览", + "dataAssetsIndex": "数据资产目录", + "overallMeasurement": "总体计量", + "menu": "目录", + "pleaseEnterATableName": "请输入 [数据库名/表名/字段名/表描述] 来模糊搜索数据表", + "search": "搜索" + } + } +} diff --git a/web/packages/dataGovernance/module/common/eventBus/event-bus.js b/web/packages/dataGovernance/module/common/eventBus/event-bus.js new file mode 100644 index 000000000..c5dbc39b1 --- /dev/null +++ b/web/packages/dataGovernance/module/common/eventBus/event-bus.js @@ -0,0 +1,3 @@ +// event-bus.js +import Vue from 'vue' +export const EventBus = new Vue() diff --git a/web/packages/dataGovernance/module/common/iCard.vue b/web/packages/dataGovernance/module/common/iCard.vue new file mode 100644 index 000000000..3cba48960 --- /dev/null +++ b/web/packages/dataGovernance/module/common/iCard.vue @@ -0,0 +1,76 @@ + + + + + diff --git a/web/packages/dataGovernance/module/common/navMenu.vue b/web/packages/dataGovernance/module/common/navMenu.vue new file mode 100644 index 000000000..ff6c2f21d --- /dev/null +++ b/web/packages/dataGovernance/module/common/navMenu.vue @@ -0,0 +1,152 @@ + + + diff --git a/web/packages/dataGovernance/module/common/tabCard/index.vue b/web/packages/dataGovernance/module/common/tabCard/index.vue new file mode 100644 index 000000000..64220e6c5 --- /dev/null +++ b/web/packages/dataGovernance/module/common/tabCard/index.vue @@ -0,0 +1,218 @@ + + + + + diff --git a/web/packages/dataGovernance/module/common/title.vue b/web/packages/dataGovernance/module/common/title.vue new file mode 100644 index 000000000..7650c51cc --- /dev/null +++ b/web/packages/dataGovernance/module/common/title.vue @@ -0,0 +1,42 @@ + + + + + diff --git a/web/packages/dataGovernance/module/common/tree/tree-item.vue b/web/packages/dataGovernance/module/common/tree/tree-item.vue new file mode 100644 index 000000000..56584c2a3 --- /dev/null +++ b/web/packages/dataGovernance/module/common/tree/tree-item.vue @@ -0,0 +1,192 @@ + + + diff --git a/web/packages/dataGovernance/module/common/tree/tree.vue b/web/packages/dataGovernance/module/common/tree/tree.vue new file mode 100644 index 000000000..2387e3116 --- /dev/null +++ b/web/packages/dataGovernance/module/common/tree/tree.vue @@ -0,0 +1,121 @@ + + diff --git a/web/packages/dataGovernance/module/common/treeMenu.vue b/web/packages/dataGovernance/module/common/treeMenu.vue new file mode 100644 index 000000000..ea6871f54 --- /dev/null +++ b/web/packages/dataGovernance/module/common/treeMenu.vue @@ -0,0 +1,200 @@ + + + diff --git a/web/packages/dataGovernance/module/components/tabCard/index.vue b/web/packages/dataGovernance/module/components/tabCard/index.vue new file mode 100644 index 000000000..cf90d7c2a --- /dev/null +++ b/web/packages/dataGovernance/module/components/tabCard/index.vue @@ -0,0 +1,83 @@ + + + + + diff --git a/web/packages/dataGovernance/module/dataGovernance/assetsIndex.vue b/web/packages/dataGovernance/module/dataGovernance/assetsIndex.vue new file mode 100644 index 000000000..50c050018 --- /dev/null +++ b/web/packages/dataGovernance/module/dataGovernance/assetsIndex.vue @@ -0,0 +1,235 @@ + + + diff --git a/web/packages/dataGovernance/module/dataGovernance/index.js b/web/packages/dataGovernance/module/dataGovernance/index.js new file mode 100644 index 000000000..332e4368e --- /dev/null +++ b/web/packages/dataGovernance/module/dataGovernance/index.js @@ -0,0 +1,10 @@ +export default { + name: 'dataGovernance', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `${location.protocol}//${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/packages/dataGovernance/module/dataGovernance/index.vue b/web/packages/dataGovernance/module/dataGovernance/index.vue new file mode 100644 index 000000000..c3707291b --- /dev/null +++ b/web/packages/dataGovernance/module/dataGovernance/index.vue @@ -0,0 +1,148 @@ + + + diff --git a/web/packages/dataGovernance/module/dataGovernance/overview.vue b/web/packages/dataGovernance/module/dataGovernance/overview.vue new file mode 100644 index 000000000..f0edb970c --- /dev/null +++ b/web/packages/dataGovernance/module/dataGovernance/overview.vue @@ -0,0 +1,309 @@ + + + diff --git a/web/packages/dataGovernance/module/header/index.js b/web/packages/dataGovernance/module/header/index.js new file mode 100644 index 000000000..4e0a78d62 --- /dev/null +++ b/web/packages/dataGovernance/module/header/index.js @@ -0,0 +1,5 @@ +export default { + name: 'scriptisHeader', + component: () => import('./index.vue'), + dispatchs: [], +}; \ No newline at end of file diff --git a/web/packages/dataGovernance/module/header/index.scss b/web/packages/dataGovernance/module/header/index.scss new file mode 100644 index 000000000..fd86b1b0a --- /dev/null +++ b/web/packages/dataGovernance/module/header/index.scss @@ -0,0 +1,21 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +@import '@dataspherestudio/shared/common/style/variables.scss'; + +@import '@dataspherestudio/shared/common/style/headerCommon.scss'; + diff --git a/web/packages/dataGovernance/module/header/index.vue b/web/packages/dataGovernance/module/header/index.vue new file mode 100644 index 000000000..9ce37909e --- /dev/null +++ b/web/packages/dataGovernance/module/header/index.vue @@ -0,0 +1,102 @@ + + + diff --git a/web/packages/dataGovernance/module/header/userMenu.vue b/web/packages/dataGovernance/module/header/userMenu.vue new file mode 100644 index 000000000..38e34ca8b --- /dev/null +++ b/web/packages/dataGovernance/module/header/userMenu.vue @@ -0,0 +1,109 @@ + + + diff --git a/web/packages/dataGovernance/package.json b/web/packages/dataGovernance/package.json new file mode 100644 index 000000000..4be406d69 --- /dev/null +++ b/web/packages/dataGovernance/package.json @@ -0,0 +1,9 @@ +{ + "name": "@dataspherestudio/dataGovernance", + "version": "1.1.4", + "dependencies": { + "@dataspherestudio/shared": "^1.1.4", + "clipboard": "2.0.8", + "jquery": "3.6.0" + } +} diff --git a/web/packages/dataGovernance/router.js b/web/packages/dataGovernance/router.js new file mode 100644 index 000000000..c29c1bbfa --- /dev/null +++ b/web/packages/dataGovernance/router.js @@ -0,0 +1,86 @@ +const routes = [ + { + path: '/dataGovernance', + name: 'dataGovernance', + meta: { + title: '数据治理', + publicPage: true + }, + component: () => import('./view/governance/index.vue'), + redirect: '/dataGovernance/overview', + children: [{ + path: 'overview', + name: 'dataGovernance/overview', + meta: { + title: '数据总览', + publicPage: true + }, + component: () => import('./module/dataGovernance/overview.vue'), + },{ + path: 'assets', + name: 'dataGovernance/assets', + meta: { + title: '数据资产目录', + publicPage: true + }, + component: () => import('./module/dataGovernance/assetsIndex.vue'), + redirect: '/dataGovernance/assets/search', + children: [{ + path: 'search', + name: 'dataGovernance/assets/search', + meta: { + title: '数据资产目录查询', + publicPage: true + }, + component: () => import('./view/assetsSearch/index.vue'), + },{ + path: 'info/:guid', + name: 'dataGovernance/assets/info', + meta: { + title: '数据资产详情', + publicPage: true + }, + component: () => import('./view/assetsInfo/index.vue'), + }] + },{ + name: 'dataGovernance/subjectDomain', + path: 'subjectDomain', + component: () => import('./view/subjectDomain/index.vue'), + meta: { + title: '主题域配置', + publicPage: true, + icon: 'ios-paper' + } + },{ + name: 'dataGovernance/layered', + path: 'layered', + component: () => import('./view/layered/index.vue'), + meta: { + title: '分层配置', + publicPage: true, + icon: 'ios-paper' + } + },{ + name: 'dataGovernance/modifier', + path: 'modifier', + component: () => import('./view/modifier/index.vue'), + meta: { + title: '修饰词管理', + publicPage: true, + icon: 'ios-paper' + } + }, + { + name: 'dataGovernance/statPeriod', + path: 'statPeriod', + component: () => import('./view/statPeriod/index.vue'), + meta: { + title: '统计周期管理', + publicPage: true, + icon: 'ios-paper' + } + }] + } +] + +export default routes; diff --git a/web/packages/dataGovernance/service/api.js b/web/packages/dataGovernance/service/api.js new file mode 100644 index 000000000..0a81fbbb1 --- /dev/null +++ b/web/packages/dataGovernance/service/api.js @@ -0,0 +1,504 @@ +import API_PATH from '@dataspherestudio/shared/common/config/apiPath.js' +import api from '@dataspherestudio/shared/common/service/api' + +/** + * 数据资产概要 + * @returns {Object.result} + * + */ +export const getHiveSummary = () => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveSummary`, 'get') + +/** + * 查询hive表--基础&列 + * @param {*} guid + * @returns + */ +export const getHiveTblBasic = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/basic`, {}, 'get') + +/** + * 查询hive表-分区信息 + * @param {*} guid + * @returns + */ +export const getHiveTblPartition = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/partition`, {}, 'get') + +/** + * 查询hive表--select语句 + * @param {*} guid + * @returns + */ +export const getSelectSql = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/select`, {}, 'get') + +/** + * 查询hive表--create语句 + * @param {*} guid + * @returns + */ +export const getSelectDdl = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/create`, {}, 'get') + +/** + * 搜索hive表 + * @param {query} + * @returns + */ +export const getHiveTbls = params => + api.fetch( + `${API_PATH.DATA_GOVERNANCE}hiveTbl/search`, + params, + 'get' + ) + +/** + * 修改主题域,分层 + * @params {guid, data} + * @returns String + */ +export const updateClassifications = (guid, data) => + api.fetch( + `${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/classifications`, + data, + 'put' + ) + + +/* + * 查询hive表--血缘 + * @param {*} guid + * @returnsL + */ +export const getLineage = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/lineage`, {}, 'get') + +/** + * 批量修改注释 + * @params {Map} + * @returns + */ +export const putCommetBulk = params => { + return api.fetch(`${API_PATH.DATA_GOVERNANCE}comment/bulk`, params, 'put') +} + +/** + * 存储量前10表 + * @params {void} + * @returns Array + */ +export const getTopStorage = () => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/topStorage`, {}, 'get') + +/** + * 设置标签--表或列 + * @params {guid} + * @returns String + */ +export const postSetLabel = (guid, params) => + api.fetch(`${API_PATH.DATA_GOVERNANCE}label/${guid}`, params, 'post') + +/** + * 删除标签 + * @params {guid} + * @returns String + */ +export const putRemoveLabel = (guid, params) => + api.fetch(`${API_PATH.DATA_GOVERNANCE}label/${guid}`, params, 'put') + +/** + * 修改注释--表或列 + * @params {guid, Obiect} + * @returns String + */ +export const postSetComment = (guid, comment) => + api.fetch( + `${API_PATH.DATA_GOVERNANCE}comment/${guid}?comment=${comment}`, + {}, + 'put' + ) + +/** + * 负责人查询 + * @params {workspaceId} + * @returns Array + */ +export const getWorkspaceUsers = (workspackId, search) => + api.fetch( + `${API_PATH.DATA_GOVERNANCE}getWorkspaceUsers/${workspackId}/${search}`, + {}, + 'get' + ) + +/** + * 查询主主题域 + * @params {workspaceId} + * @returns Array + */ +export const getThemedomains = (keyword) => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}subject/subtypes`, + { keyword }, + 'get' + ) + +/** + * 创建主体域 + * @params {workspaceId} + * @returns Array + */ +export const createThemedomains = body => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}batch`, + body, + 'post' + ) + +/** + * 删除主题域 + * @params {id} + * @returns Any + */ +export const deleteThemedomains = name => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}${name}/delete`, + {}, + 'post' + ) + +/** + * 禁用主题域 + * @params {workspaceId} + * @returns Array + */ +export const disableThemedomains = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/themedomains/${id}/disable`, + {}, + 'put' + ) + +/** + * 启用主题域 + * @params {workspaceId} + * @returns Array + */ +export const enableThemedomains = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/themedomains/${id}/enable`, + {}, + 'put' + ) + +/** + * 根据name获取主题 + * @params {workspaceId} + * @returns Array + */ +export const getThemedomainsById = name => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}${name}`, + {}, + 'get' + ) + +/** + * 编辑主题 + * @params {workspaceId} + * @returns Array + */ +export const editThemedomains = (body) => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}batch`, + body, + 'put' + ) + +/** + * 查询所有分层 + * @params {workspaceId} + * @returns Array + */ +export const getLayersAll = () => + api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/all`, {}, 'get') + +/** + * 查询所有预置分层 + * @params {workspaceId} + * @returns Array + */ +export const getLayersPreset = () => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}layer_system/subtypes`, + {}, + 'get' + ) + +/** + * 查询自定义分层 + * @params {workspaceId} + * @returns Array + */ +export const getLayersCustom = (keyword) => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}layer/subtypes`, + {keyword}, + 'get' + ) + +/** + * 新增自定义分层 + * @params {workspaceId} + * @returns Array + */ +export const createLayersCustom = body => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}batch`, + body, + 'post' + ) + +/** + * 根据ID查询某个分层信息 + * @params {workspaceId} + * @returns Array + */ +export const getLayersById = name => + api.fetch(`${API_PATH.WAREHOUSE_PATH}${name}`, {}, 'get') + +/** + * 编辑分层 + * @params {workspaceId} + * @returns Array + */ +export const editLayersCustom = (body) => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}batch`, + body, + 'put' + ) + +/** + * 删除分层 + * @params {workspaceId} + * @returns Array + */ +export const deleteLayers = name => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}${name}/delete`, + {}, + 'post' + ) + +/** + * 禁用分层 + * @params {workspaceId} + * @returns Array + */ +export const disableLayers = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/layers/${id}/disable`, + {}, + 'put' + ) + +/** + * 启用分层 + * @params {workspaceId} + * @returns Array + */ +export const enableLayers = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/layers/${id}/enable`, + {}, + 'put' + ) + +/** + * 创建修饰词 + * @params {workspaceId} + * @returns Array + */ +export const createModifiers = body => + api.fetch(`${API_PATH.WAREHOUSE_PATH}data-warehouse/modifiers`, body, 'post') + +/** + * 删除修饰词 + * @params {workspaceId} + * @returns Array + */ +export const deleteModifiers = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/modifiers/${id}`, + {}, + 'delete' + ) + +/** + * 编辑修饰词 + * @params {workspaceId} + * @returns Array + */ +export const editModifiers = (id, body) => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/modifiers/${id}`, + body, + 'put' + ) + +/** + * 分页获取修饰词 + * @params {workspaceId} + * @returns Array + */ +export const getModifiers = (page, size, name, enabled) => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/modifiers`, + { page, size, name, enabled }, + 'get' + ) + +/** + * 根据id获取 + * @params {workspaceId} + * @returns Array + */ +export const getModifiersById = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/modifiers/${id}`, + {}, + 'get' + ) + +/** + * 禁用修饰词 + * @params {workspaceId} + * @returns Array + */ +export const disableModifiers = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/modifiers/${id}/disable`, + {}, + 'put' + ) + +/** + * 启用修饰词 + * @params {workspaceId} + * @returns Array + */ +export const enableModifiers = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/modifiers/${id}/enable`, + {}, + 'put' + ) + +//////// + +/** + * 创建统计周期 + * @params {workspaceId} + * @returns Array + */ +export const createStatisticalPeriods = body => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/statistical_periods`, + body, + 'post' + ) + +/** + * 删除统计周期 + * @params {workspaceId} + * @returns Array + */ +export const deleteStatisticalPeriods = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/statistical_periods/${id}`, + {}, + 'delete' + ) + +/** + * 编辑统计周期 + * @params {workspaceId} + * @returns Array + */ +export const editStatisticalPeriods = (id, body) => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/statistical_periods/${id}`, + body, + 'put' + ) + +/** + * 分页获取统计周期 + * @params {workspaceId} + * @returns Array + */ +export const getStatisticalPeriods = (page, size, name, enabled) => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/statistical_periods`, + { page, size, name, enabled }, + 'get' + ) + +/** + * 根据id获取统计周期 + * @params {workspaceId} + * @returns Array + */ +export const getStatisticalPeriodsById = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/statistical_periods/${id}`, + {}, + 'get' + ) + +/** + * 禁用统计周期 + * @params {workspaceId} + * @returns Array + */ +export const disableStatisticalPeriods = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/statistical_periods/${id}/disable`, + {}, + 'put' + ) + +/** + * 启用统计周期 + * @params {workspaceId} + * @returns Array + */ +export const enableStatisticalPeriods = id => + api.fetch( + `${API_PATH.WAREHOUSE_PATH}data-warehouse/statistical_periods/${id}/enable`, + {}, + 'put' + ) + +// export { +// getHiveSummary, +// getHiveTblBasic, +// getHiveTblPartition, +// getSelectSql, +// getSelectDdl, +// getHiveTbls, +// getLineage, +// putCommetBulk, +// getTopStorage, +// postSetLabel, +// postSetComment, +// getWorkspaceUsers, +// getThemedomains, +// createThemedomains, +// deleteThemedomains, +// disableThemedomains, +// enableThemedomains, +// getThemedomainsById, +// editThemedomains +// } diff --git a/web/packages/dataGovernance/service/db/node.js b/web/packages/dataGovernance/service/db/node.js new file mode 100644 index 000000000..6347b2c21 --- /dev/null +++ b/web/packages/dataGovernance/service/db/node.js @@ -0,0 +1,103 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import {isNil} from 'lodash'; +import { Basic } from '@dataspherestudio/shared/common/service/db/index.js'; +import { isArray } from 'util'; +/** + * @class Globalcache + * @extends {Basic} + */ +class Node extends Basic { + /** + *Creates an instance of Globalcache. + * @param {*} table + * @param {*} db + * @memberof Globalcache + */ + constructor(table) { + super(table); + } + + /** + * @param {*} nodeId + * @param {*} key + * @return {*} + */ + async getNodeCache({nodeId, key}) { + let caches = await this.get(nodeId); + if (key) { + return caches[0][key]; + } + return caches[0]; + } + + /** + * @param {*} nodeId + * @param {*} value + * @return {*} + */ + async addNodeCache({nodeId, value}) { + let cache = await this.getNodeCache({nodeId}); + if (!isNil(cache)) { + return this.update(nodeId, value) + } + return this.add(Object.assign({nodeId}, value)); + } + + + /** + * @param {*} nodeId + * @param {*} key + * @param {*} value + * @return {*} + */ + async updateNodeCache({nodeId, key, value}) { + let cache = {}; + if (key) { + if (isArray(key)) { + key.forEach((k) => { + cache[k] = value[k]; + }) + } else { + cache[key] = value[key]; + } + } else { + cache = value; + } + return nodeId ? this.update(nodeId, cache) : null; + } + + + /** + * @param {*} args + * @return {*} + */ + async removeNodeCache({nodeId, key}) { + let cache = await this.getNodeCache({nodeId}); + if (isNil(cache)) { + if (key) { + cache[key] = null; + return this.update(nodeId, cache); + } + return this.remove(nodeId); + } + } +} +const node = new Node('node'); + +export default node; diff --git a/web/packages/dataGovernance/service/db/project.js b/web/packages/dataGovernance/service/db/project.js new file mode 100644 index 000000000..760a6636e --- /dev/null +++ b/web/packages/dataGovernance/service/db/project.js @@ -0,0 +1,118 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import {isNil} from 'lodash'; +import { Basic } from '@dataspherestudio/shared/common/service/db/index.js'; +import { isArray } from 'util'; +/** + * @class Globalcache + * @extends {Basic} + */ +class Project extends Basic { + /** + *Creates an instance of Globalcache. + * @param {*} table + * @param {*} db + * @memberof Globalcache + */ + constructor(table) { + super(table); + } + + /** + * @param {*} projectID + * @param {*} key + * @return {*} + */ + async getProjectCache({projectID, key}) { + projectID = Number(projectID); + let caches = await this.get(projectID); + if (key) { + return caches[0][key]; + } + return caches[0]; + } + + /** + * @param {*} projectID + * @param {*} value + * @return {*} + */ + async addProjectCache({projectID, value}) { + projectID = Number(projectID); + let cache = await this.getProjectCache({projectID}); + if (!isNil(cache)) { + return this.update(projectID, value) + } + return this.add(Object.assign({projectID}, value)); + } + + + /** + * @param {*} projectID + * @param {*} key + * @param {*} value token是用于第二层比对的字段,sKey是实际写入的key,sValue是用于比对的值 + * @param {*} isDeep 是否要对第二层数据进行修改,如果需要的话,就需要在value中携带 + * @return {*} + */ + async updateProjectCache({projectID, key, value, isDeep}) { + projectID = Number(projectID); + let cache = {}; + if (key) { + if (isDeep) { + cache = await this.getProjectCache({projectID}) || {}; + let arr = cache && cache[key] || []; + arr.forEach((item) => { + if (item[value.token] === Number(value.sValue)) { + item[value.sKey] = value[value.sKey]; + } + }) + } else { + if (isArray(key)) { + key.forEach((k) => { + cache[k] = value[k]; + }) + } else { + cache[key] = value[key]; + } + } + } else { + cache = value; + } + return this.update(projectID, cache); + } + + + /** + * @param {*} args + * @return {*} + */ + async removeProjectCache({projectID, key}) { + projectID = Number(projectID); + let cache = await this.getProjectCache({projectID}); + if (isNil(cache)) { + if (key) { + cache[key] = null; + return this.update(projectID, cache); + } + return this.remove(projectID); + } + } +} +const project = new Project('project'); + +export default project; diff --git a/web/packages/dataGovernance/utils/clipboard.js b/web/packages/dataGovernance/utils/clipboard.js new file mode 100644 index 000000000..7efc01c49 --- /dev/null +++ b/web/packages/dataGovernance/utils/clipboard.js @@ -0,0 +1,43 @@ +import Vue from 'vue'; +import Clipboard from 'clipboard'; + +/** + *@returns + */ +function clipboardSuccess() { + Vue.prototype.$Message.success({ + content: '复制成功', + duration: 1.5 + }); +} + + +/** + *@returns + */ +function clipboardError() { + Vue.prototype.$Message.success({ + message: '复制失败', + duration: 1.5 + }); +} + +/** + * 把字符串中的内容放到剪切板上 + * @param {String, Object} + * @param {void} + */ +export default function handleClipboard(text, event) { + const clipboard = new Clipboard(event.target, { + text: () => text + }); + clipboard.on('success', () => { + clipboardSuccess(); + clipboard.destroy(); + }); + clipboard.on('error', () => { + clipboardError(); + clipboard.destroy(); + }); + clipboard.onClick(event); +} diff --git a/web/packages/dataGovernance/utils/fomatSQL.js b/web/packages/dataGovernance/utils/fomatSQL.js new file mode 100644 index 000000000..d69db0d85 --- /dev/null +++ b/web/packages/dataGovernance/utils/fomatSQL.js @@ -0,0 +1,24 @@ +/** + * 转换 后端SQL语句 成 html文本 + * @param (String) + * @return (String) + */ +export function fomatSqlForShow(sql) { + if (typeof sql !== "string") return ""; + const arr = sql.split("@$"); + const res = arr.map((i, idx) => `

${idx + 1}${i}

`); + return res.join(""); +} + +/** + * + * @param {String} sql + * @return {String} + */ +export function fomatSqlForCopy(sql) { + if (typeof sql !== "string") return ""; + return sql.replaceAll("@$", "\r\n"); + // const arr = sql.replaceAll(" ", "").split("@$"); + // const res = arr.map(i => `${i}\r\n`); + // return res.join(""); +} diff --git a/web/packages/dataGovernance/utils/formatDate.js b/web/packages/dataGovernance/utils/formatDate.js new file mode 100644 index 000000000..d3d68b595 --- /dev/null +++ b/web/packages/dataGovernance/utils/formatDate.js @@ -0,0 +1,11 @@ +import moment from 'moment' + +/** + * 转换时间戳到字符串 + * @param (Number) + * @return (String) + */ +export default function formatDate (value) { + if (!value) return '' + return moment(value).format("YYYY-MM-DD HH:mm") +} diff --git a/web/packages/dataGovernance/utils/storage.js b/web/packages/dataGovernance/utils/storage.js new file mode 100644 index 000000000..2ab293c3b --- /dev/null +++ b/web/packages/dataGovernance/utils/storage.js @@ -0,0 +1,11 @@ +export const storage = { + setItem(key, val) { + return sessionStorage.setItem(key, val) + }, + getItem(key) { + return sessionStorage.getItem(key) + }, + removeItem(key) { + return sessionStorage.removeItem(key) + } +} diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/base_node.js b/web/packages/dataGovernance/view/assetsInfo/components/lineage/base_node.js new file mode 100644 index 000000000..ab46f8364 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/base_node.js @@ -0,0 +1,42 @@ +/* eslint-disable */ +import {Node} from 'butterfly-dag'; + +import './base_node.scss'; +import './symbol/iconfont' +import util from '@dataspherestudio/shared/common/util' + +class BaseNode extends Node { + constructor(opts) { + super(opts); + this.options = opts; + } + + draw = (opts) => { + let container = $('
') + .css('top', opts.top) + .css('left', opts.left) + .attr('id', opts.id) + .addClass(opts.options.className); + let text = `${opts.options.name}` + if (opts.options.status !== 'ACTIVE') { + text = `${opts.options.name}` + } else if (opts.options.icon === 'icon-a-hivetable') { + text = `${opts.options.name}` + util.Hub.$emit('register_click_hive_table', opts.options.model) + } + util.Hub.$emit('register_hover', opts.options.model) + let logoContainer = + $(`
+ + ${text}
`); + logoContainer.addClass(opts.options.className); + + container.append(logoContainer); + + return container[0]; + } +} + +export default BaseNode; diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/base_node.scss b/web/packages/dataGovernance/view/assetsInfo/components/lineage/base_node.scss new file mode 100644 index 000000000..ff34f2a80 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/base_node.scss @@ -0,0 +1,53 @@ +.dagreLayout-page { + .relation-node { + position: absolute; + width: 140px; + height: 36px; + padding: 0; + box-shadow:0 2px 3px 0 rgba(0,112,204,0.06); + border-radius: 100px; + border:1px solid #D9D9D9; + cursor: pointer; + background-color: white !important; + &:hover { + border: 1px solid #2E92F7; + .datac-icon { + line-height: 52px; + } + } + .logo-container { + width: 100%; + height: 100%; + border: none; + line-height: 36px; + text-align: center; + margin: auto; + display: inherit; + } + } + + .datac-icon { + font-size: 35px; + line-height: 60px; + } + .long-text { + text-align: center; + } + .nodeBackground-color{ + background-color: rgba(255, 255, 255, 0.8); + max-width: 300px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + } + .current-bg-color { + background-color: aliceblue; + } + .should-hide { + fill: none !important; + stroke: none !important; + } + .should-lower { + opacity: 0.2; + } +} diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/dagre-canvas.js b/web/packages/dataGovernance/view/assetsInfo/components/lineage/dagre-canvas.js new file mode 100644 index 000000000..eacf9021f --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/dagre-canvas.js @@ -0,0 +1,47 @@ +import _ from 'lodash'; +import {Canvas, Layout} from 'butterfly-dag'; + +class DagreCanvas extends Canvas { + drageReDraw(newParam) { + let {nodes, layout, edges} = this; + let addResultNodes = nodes.map((item) => { + return item.options; + }); + if (newParam) { + layout.options = { + ...layout.options, + ...newParam + }; + } + Layout.dagreLayout({ + rankdir: (newParam && newParam.rankdir) || _.get(layout, 'options.rankdir') || 'TB', + align: (newParam && newParam.align) || _.get(layout, 'options.align'), + nodeSize: (newParam && newParam.nodeSize) || _.get(layout, 'options.nodeSize'), + nodesepFunc: (newParam && newParam.nodesepFunc) || _.get(layout, 'options.nodesepFunc'), + ranksepFunc: (newParam && newParam.ranksepFunc) || _.get(layout, 'options.ranksepFunc'), + nodesep: (newParam && newParam.nodesep) || _.get(layout, 'options.nodesep') || 50, + ranksep: (newParam && newParam.ranksep) || _.get(layout, 'options.ranksep') || 50, + controlPoints: (newParam && newParam.controlPoints) || _.get(layout, 'options.controlPoints') || false, + data: { + nodes: addResultNodes, + edges: edges.map(item => ({ + source: item.sourceNode.id, + target: item.targetNode.id + })) + }}); + // 布局计算完位置后left和top赋值给node节点 + nodes.forEach((item, index) => { + let currentNodeNewLeft = addResultNodes[index].left; + let currentNodeNewTop = addResultNodes[index].top; + let currentNodeNewPosInfo = addResultNodes[index].posInfo; + if (item.top !== currentNodeNewTop || item.left !== currentNodeNewLeft) { + item.options.top = currentNodeNewTop; + item.options.left = currentNodeNewLeft; + item.options.posInfo = currentNodeNewPosInfo; + item.moveTo(currentNodeNewLeft, currentNodeNewTop); + } + }); + } +} + +export default DagreCanvas; diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/edge.js b/web/packages/dataGovernance/view/assetsInfo/components/lineage/edge.js new file mode 100644 index 000000000..a98f650d1 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/edge.js @@ -0,0 +1,31 @@ +import $ from 'jquery'; +import {Edge} from 'butterfly-dag'; + +class RelationEdge extends Edge { + draw(obj) { + let path = super.draw(obj); + if (this.options.color) { + $(path).addClass(this.options.color); + } + $(path).attr('id', this.options.id) + return path; + } + + drawArrow(isShow) { + let dom = super.drawArrow(isShow); + if (this.options.color) { + $(dom).addClass(this.options.color); + } + return dom; + } + + drawLabel(text) { + let dom = null; + if (text) { + dom = $(`${text}`)[0]; + } + return dom; + } +} + +export default RelationEdge; diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.js b/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.js new file mode 100644 index 000000000..6e9946229 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.js @@ -0,0 +1,2 @@ +import lineage from './index.vue' +export default lineage; diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.scss b/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.scss new file mode 100644 index 000000000..d5a783420 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.scss @@ -0,0 +1,47 @@ +.dagreLayout-page{ + width: 100%; + height: 100%; + position: relative; + .flow-canvas { + width: 100%; + height: 100%; + } + .point { + position: absolute; + width: 8px; + height: 8px; + border-radius: 50%; + border: 1px solid #D9D9D9; + cursor: pointer; + } + .point-1 { + top: 50px; + left: 300px; + } + .point-2 { + top: 300px; + left: 500px; + } + .operate-bar{ + position: absolute; + top: 20px; + right: 20px; + z-index: 999; + box-shadow:0 2px 3px 0 rgba(0,112,204,0.06); + border:1px solid #D9D9D9; + border-radius: 4px; + padding: 10px; + background-color: rgba(255, 255, 255, 0.8); + color:#222; + .operate-bar-title{ + height: 30px; + width: 100%; + line-height: 30px; + text-align: center; + background-color: #F66902; + color: #222; + margin-bottom: 10px; + } + + } +} diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.vue b/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.vue new file mode 100644 index 000000000..ba1b816d4 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/index.vue @@ -0,0 +1,120 @@ + + diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/demo.css b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/demo.css new file mode 100644 index 000000000..a67054a0a --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/demo.css @@ -0,0 +1,539 @@ +/* Logo 字体 */ +@font-face { + font-family: "iconfont logo"; + src: url('https://at.alicdn.com/t/font_985780_km7mi63cihi.eot?t=1545807318834'); + src: url('https://at.alicdn.com/t/font_985780_km7mi63cihi.eot?t=1545807318834#iefix') format('embedded-opentype'), + url('https://at.alicdn.com/t/font_985780_km7mi63cihi.woff?t=1545807318834') format('woff'), + url('https://at.alicdn.com/t/font_985780_km7mi63cihi.ttf?t=1545807318834') format('truetype'), + url('https://at.alicdn.com/t/font_985780_km7mi63cihi.svg?t=1545807318834#iconfont') format('svg'); +} + +.logo { + font-family: "iconfont logo"; + font-size: 160px; + font-style: normal; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +/* tabs */ +.nav-tabs { + position: relative; +} + +.nav-tabs .nav-more { + position: absolute; + right: 0; + bottom: 0; + height: 42px; + line-height: 42px; + color: #666; +} + +#tabs { + border-bottom: 1px solid #eee; +} + +#tabs li { + cursor: pointer; + width: 100px; + height: 40px; + line-height: 40px; + text-align: center; + font-size: 16px; + border-bottom: 2px solid transparent; + position: relative; + z-index: 1; + margin-bottom: -1px; + color: #666; +} + + +#tabs .active { + border-bottom-color: #f00; + color: #222; +} + +.tab-container .content { + display: none; +} + +/* 页面布局 */ +.main { + padding: 30px 100px; + width: 960px; + margin: 0 auto; +} + +.main .logo { + color: #333; + text-align: left; + margin-bottom: 30px; + line-height: 1; + height: 110px; + margin-top: -50px; + overflow: hidden; + *zoom: 1; +} + +.main .logo a { + font-size: 160px; + color: #333; +} + +.helps { + margin-top: 40px; +} + +.helps pre { + padding: 20px; + margin: 10px 0; + border: solid 1px #e7e1cd; + background-color: #fffdef; + overflow: auto; +} + +.icon_lists { + width: 100% !important; + overflow: hidden; + *zoom: 1; +} + +.icon_lists li { + width: 100px; + margin-bottom: 10px; + margin-right: 20px; + text-align: center; + list-style: none !important; + cursor: default; +} + +.icon_lists li .code-name { + line-height: 1.2; +} + +.icon_lists .icon { + display: block; + height: 100px; + line-height: 100px; + font-size: 42px; + margin: 10px auto; + color: #333; + -webkit-transition: font-size 0.25s linear, width 0.25s linear; + -moz-transition: font-size 0.25s linear, width 0.25s linear; + transition: font-size 0.25s linear, width 0.25s linear; +} + +.icon_lists .icon:hover { + font-size: 100px; +} + +.icon_lists .svg-icon { + /* 通过设置 font-size 来改变图标大小 */ + width: 1em; + /* 图标和文字相邻时,垂直对齐 */ + vertical-align: -0.15em; + /* 通过设置 color 来改变 SVG 的颜色/fill */ + fill: currentColor; + /* path 和 stroke 溢出 viewBox 部分在 IE 下会显示 + normalize.css 中也包含这行 */ + overflow: hidden; +} + +.icon_lists li .name, +.icon_lists li .code-name { + color: #666; +} + +/* markdown 样式 */ +.markdown { + color: #666; + font-size: 14px; + line-height: 1.8; +} + +.highlight { + line-height: 1.5; +} + +.markdown img { + vertical-align: middle; + max-width: 100%; +} + +.markdown h1 { + color: #404040; + font-weight: 500; + line-height: 40px; + margin-bottom: 24px; +} + +.markdown h2, +.markdown h3, +.markdown h4, +.markdown h5, +.markdown h6 { + color: #404040; + margin: 1.6em 0 0.6em 0; + font-weight: 500; + clear: both; +} + +.markdown h1 { + font-size: 28px; +} + +.markdown h2 { + font-size: 22px; +} + +.markdown h3 { + font-size: 16px; +} + +.markdown h4 { + font-size: 14px; +} + +.markdown h5 { + font-size: 12px; +} + +.markdown h6 { + font-size: 12px; +} + +.markdown hr { + height: 1px; + border: 0; + background: #e9e9e9; + margin: 16px 0; + clear: both; +} + +.markdown p { + margin: 1em 0; +} + +.markdown>p, +.markdown>blockquote, +.markdown>.highlight, +.markdown>ol, +.markdown>ul { + width: 80%; +} + +.markdown ul>li { + list-style: circle; +} + +.markdown>ul li, +.markdown blockquote ul>li { + margin-left: 20px; + padding-left: 4px; +} + +.markdown>ul li p, +.markdown>ol li p { + margin: 0.6em 0; +} + +.markdown ol>li { + list-style: decimal; +} + +.markdown>ol li, +.markdown blockquote ol>li { + margin-left: 20px; + padding-left: 4px; +} + +.markdown code { + margin: 0 3px; + padding: 0 5px; + background: #eee; + border-radius: 3px; +} + +.markdown strong, +.markdown b { + font-weight: 600; +} + +.markdown>table { + border-collapse: collapse; + border-spacing: 0px; + empty-cells: show; + border: 1px solid #e9e9e9; + width: 95%; + margin-bottom: 24px; +} + +.markdown>table th { + white-space: nowrap; + color: #333; + font-weight: 600; +} + +.markdown>table th, +.markdown>table td { + border: 1px solid #e9e9e9; + padding: 8px 16px; + text-align: left; +} + +.markdown>table th { + background: #F7F7F7; +} + +.markdown blockquote { + font-size: 90%; + color: #999; + border-left: 4px solid #e9e9e9; + padding-left: 0.8em; + margin: 1em 0; +} + +.markdown blockquote p { + margin: 0; +} + +.markdown .anchor { + opacity: 0; + transition: opacity 0.3s ease; + margin-left: 8px; +} + +.markdown .waiting { + color: #ccc; +} + +.markdown h1:hover .anchor, +.markdown h2:hover .anchor, +.markdown h3:hover .anchor, +.markdown h4:hover .anchor, +.markdown h5:hover .anchor, +.markdown h6:hover .anchor { + opacity: 1; + display: inline-block; +} + +.markdown>br, +.markdown>p>br { + clear: both; +} + + +.hljs { + display: block; + background: white; + padding: 0.5em; + color: #333333; + overflow-x: auto; +} + +.hljs-comment, +.hljs-meta { + color: #969896; +} + +.hljs-string, +.hljs-variable, +.hljs-template-variable, +.hljs-strong, +.hljs-emphasis, +.hljs-quote { + color: #df5000; +} + +.hljs-keyword, +.hljs-selector-tag, +.hljs-type { + color: #a71d5d; +} + +.hljs-literal, +.hljs-symbol, +.hljs-bullet, +.hljs-attribute { + color: #0086b3; +} + +.hljs-section, +.hljs-name { + color: #63a35c; +} + +.hljs-tag { + color: #333333; +} + +.hljs-title, +.hljs-attr, +.hljs-selector-id, +.hljs-selector-class, +.hljs-selector-attr, +.hljs-selector-pseudo { + color: #795da3; +} + +.hljs-addition { + color: #55a532; + background-color: #eaffea; +} + +.hljs-deletion { + color: #bd2c00; + background-color: #ffecec; +} + +.hljs-link { + text-decoration: underline; +} + +/* 代码高亮 */ +/* PrismJS 1.15.0 +https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ +/** + * prism.js default theme for JavaScript, CSS and HTML + * Based on dabblet (http://dabblet.com) + * @author Lea Verou + */ +code[class*="language-"], +pre[class*="language-"] { + color: black; + background: none; + text-shadow: 0 1px white; + font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; + text-align: left; + white-space: pre; + word-spacing: normal; + word-break: normal; + word-wrap: normal; + line-height: 1.5; + + -moz-tab-size: 4; + -o-tab-size: 4; + tab-size: 4; + + -webkit-hyphens: none; + -moz-hyphens: none; + -ms-hyphens: none; + hyphens: none; +} + +pre[class*="language-"]::-moz-selection, +pre[class*="language-"] ::-moz-selection, +code[class*="language-"]::-moz-selection, +code[class*="language-"] ::-moz-selection { + text-shadow: none; + background: #b3d4fc; +} + +pre[class*="language-"]::selection, +pre[class*="language-"] ::selection, +code[class*="language-"]::selection, +code[class*="language-"] ::selection { + text-shadow: none; + background: #b3d4fc; +} + +@media print { + + code[class*="language-"], + pre[class*="language-"] { + text-shadow: none; + } +} + +/* Code blocks */ +pre[class*="language-"] { + padding: 1em; + margin: .5em 0; + overflow: auto; +} + +:not(pre)>code[class*="language-"], +pre[class*="language-"] { + background: #f5f2f0; +} + +/* Inline code */ +:not(pre)>code[class*="language-"] { + padding: .1em; + border-radius: .3em; + white-space: normal; +} + +.token.comment, +.token.prolog, +.token.doctype, +.token.cdata { + color: slategray; +} + +.token.punctuation { + color: #999; +} + +.namespace { + opacity: .7; +} + +.token.property, +.token.tag, +.token.boolean, +.token.number, +.token.constant, +.token.symbol, +.token.deleted { + color: #905; +} + +.token.selector, +.token.attr-name, +.token.string, +.token.char, +.token.builtin, +.token.inserted { + color: #690; +} + +.token.operator, +.token.entity, +.token.url, +.language-css .token.string, +.style .token.string { + color: #9a6e3a; + background: hsla(0, 0%, 100%, .5); +} + +.token.atrule, +.token.attr-value, +.token.keyword { + color: #07a; +} + +.token.function, +.token.class-name { + color: #DD4A68; +} + +.token.regex, +.token.important, +.token.variable { + color: #e90; +} + +.token.important, +.token.bold { + font-weight: bold; +} + +.token.italic { + font-style: italic; +} + +.token.entity { + cursor: help; +} diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/demo_index.html b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/demo_index.html new file mode 100644 index 000000000..6b3409811 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/demo_index.html @@ -0,0 +1,2304 @@ + + + + + iconfont Demo + + + + + + + + + + + + + +
+

+ + +

+ +
+
+
    + +
  • + +
    hdfs
    +
    &#xe73a;
    +
  • + +
  • + +
    hdfs-置灰
    +
    &#xe739;
    +
  • + +
  • + +
    保存
    +
    &#xe734;
    +
  • + +
  • + +
    参数
    +
    &#xe735;
    +
  • + +
  • + +
    执行
    +
    &#xe736;
    +
  • + +
  • + +
    发布
    +
    &#xe737;
    +
  • + +
  • + +
    资源
    +
    &#xe738;
    +
  • + +
  • + +
    蓝色箭头带框
    +
    &#xe72b;
    +
  • + +
  • + +
    箭头
    +
    &#xe727;
    +
  • + +
  • + +
    hive
    +
    &#xe722;
    +
  • + +
  • + +
    shell
    +
    &#xe721;
    +
  • + +
  • + +
    copy
    +
    &#xe720;
    +
  • + +
  • + +
    leftfold
    +
    &#xe71e;
    +
  • + +
  • + +
    更多省略号
    +
    &#xe71c;
    +
  • + +
  • + +
    unfold
    +
    &#xe71d;
    +
  • + +
  • + +
    收起
    +
    &#xe71a;
    +
  • + +
  • + +
    展开
    +
    &#xe71b;
    +
  • + +
  • + +
    固定到导航栏
    +
    &#xe719;
    +
  • + +
  • + +
    search
    +
    &#xe713;
    +
  • + +
  • + +
    spark process
    +
    &#xe700;
    +
  • + +
  • + +
    hive process
    +
    &#xe701;
    +
  • + +
  • + +
    删除的 hive table
    +
    &#xe702;
    +
  • + +
  • + +
    hive table
    +
    &#xe6ff;
    +
  • + +
  • + +
    主题域配置
    +
    &#xe6f9;
    +
  • + +
  • + +
    分层配置
    +
    &#xe6fa;
    +
  • + +
  • + +
    已收藏
    +
    &#xe6f7;
    +
  • + +
  • + +
    未收藏
    +
    &#xe6f8;
    +
  • + +
  • + +
    star
    +
    &#xe6f5;
    +
  • + +
  • + +
    star-filled
    +
    &#xe6f6;
    +
  • + +
  • + +
    编辑
    +
    &#xe6f4;
    +
  • + +
  • + +
    表总数
    +
    &#xe6f1;
    +
  • + +
  • + +
    数据库总数
    +
    &#xe6f2;
    +
  • + +
  • + +
    总存储量
    +
    &#xe6f3;
    +
  • + +
  • + +
    工作流
    +
    &#xe6f0;
    +
  • + +
  • + +
    新增
    +
    &#xe6ef;
    +
  • + +
  • + +
    日期
    +
    &#xe6ee;
    +
  • + +
  • + +
    组件接入管理
    +
    &#xe6eb;
    +
  • + +
  • + +
    控制台
    +
    &#xe6ec;
    +
  • + +
  • + +
    部门和用户管理
    +
    &#xe6ed;
    +
  • + +
  • + +
    电信 5G天翼云纯色logo
    +
    &#xe6c7;
    +
  • + +
  • + +
    +
    &#xe6bb;
    +
  • + +
  • + +
    下拉
    +
    &#xe6bc;
    +
  • + +
  • + +
    服务管理
    +
    &#xe6bd;
    +
  • + +
  • + +
    收起
    +
    &#xe6be;
    +
  • + +
  • + +
    总调用次数
    +
    &#xe6bf;
    +
  • + +
  • + +
    api
    +
    &#xe6c0;
    +
  • + +
  • + +
    属性
    +
    &#xe6c1;
    +
  • + +
  • + +
    API调用
    +
    &#xe6c2;
    +
  • + +
  • + +
    总执行时长用量
    +
    &#xe6c3;
    +
  • + +
  • + +
    API监控
    +
    &#xe6c4;
    +
  • + +
  • + +
    API测试
    +
    &#xe6c5;
    +
  • + +
  • + +
    API管理
    +
    &#xe6c6;
    +
  • + +
  • + +
    +
    &#xe6b3;
    +
  • + +
  • + +
    版本
    +
    &#xe6b4;
    +
  • + +
  • + +
    测试
    +
    &#xe6b5;
    +
  • + +
  • + +
    服务表
    +
    &#xe6b7;
    +
  • + +
  • + +
    服务开发
    +
    &#xe6b9;
    +
  • + +
  • + +
    新增
    +
    &#xe6ba;
    +
  • + +
  • + +
    内存卡底
    +
    &#xe6b0;
    +
  • + +
  • + +
    矩形
    +
    &#xe6af;
    +
  • + +
  • + +
    问号
    +
    &#xe6ac;
    +
  • + +
  • + +
    调度
    +
    &#xe6ad;
    +
  • + +
  • + +
    画板
    +
    &#xe6ae;
    +
  • + +
  • + +
    loading
    +
    &#xe84b;
    +
  • + +
  • + +
    file
    +
    &#xe693;
    +
  • + +
  • + +
    plus
    +
    &#xe692;
    +
  • + +
  • + +
    unfold
    +
    &#xe690;
    +
  • + +
  • + +
    put away
    +
    &#xe691;
    +
  • + +
  • + +
    open
    +
    &#xe68a;
    +
  • + +
  • + +
    delete
    +
    &#xe68c;
    +
  • + +
  • + +
    flow
    +
    &#xe68d;
    +
  • + +
  • + +
    close
    +
    &#xe68e;
    +
  • + +
  • + +
    project
    +
    &#xe68f;
    +
  • + +
  • + +
    shell
    +
    &#xe671;
    +
  • + +
  • + +
    icon-kill
    +
    &#xe667;
    +
  • + +
  • + +
    icon-ready-stop
    +
    &#xe66b;
    +
  • + +
  • + +
    icon-ready-pause
    +
    &#xe66d;
    +
  • + +
  • + +
    DELAY_EXECUTION
    +
    &#xe663;
    +
  • + +
  • + +
    NEED_FAULT_TOLERANCE
    +
    &#xe664;
    +
  • + +
  • + +
    PAUSE
    +
    &#xe665;
    +
  • + +
  • + +
    STOP
    +
    &#xe666;
    +
  • + +
  • + +
    WAITTING_DEPEND
    +
    &#xe668;
    +
  • + +
  • + +
    FORCED_SUCCES
    +
    &#xe669;
    +
  • + +
  • + +
    SUCCESS
    +
    &#xe66a;
    +
  • + +
  • + +
    SUBMITTED_SUCCES
    +
    &#xe66c;
    +
  • + +
  • + +
    FAILURE
    +
    &#xe66e;
    +
  • + +
  • + +
    RUNNING_EXECUTIO
    +
    &#xe66f;
    +
  • + +
  • + +
    WAITTING_THREAD
    +
    &#xe670;
    +
  • + +
  • + +
    REFRESH
    +
    &#xe65f;
    +
  • + +
  • + +
    FULL_SCREEN
    +
    &#xe660;
    +
  • + +
  • + +
    CANCLE_FULL_SCREEN
    +
    &#xe661;
    +
  • + +
  • + +
    DOWNLOAD
    +
    &#xe662;
    +
  • + +
+
+

Unicode 引用

+
+ +

Unicode 是字体在网页端最原始的应用方式,特点是:

+
    +
  • 支持按字体的方式去动态调整图标大小,颜色等等。
  • +
  • 默认情况下不支持多色,直接添加多色图标会自动去色。
  • +
+
+

注意:新版 iconfont 支持两种方式引用多色图标:SVG symbol 引用方式和彩色字体图标模式。(使用彩色字体图标需要在「编辑项目」中开启「彩色」选项后并重新生成。)

+
+

Unicode 使用步骤如下:

+

第一步:拷贝项目下面生成的 @font-face

+
@font-face {
+  font-family: 'iconfont';
+  src: url('iconfont.woff2?t=1642471499077') format('woff2'),
+       url('iconfont.woff?t=1642471499077') format('woff'),
+       url('iconfont.ttf?t=1642471499077') format('truetype');
+}
+
+

第二步:定义使用 iconfont 的样式

+
.iconfont {
+  font-family: "iconfont" !important;
+  font-size: 16px;
+  font-style: normal;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+

第三步:挑选相应图标并获取字体编码,应用于页面

+
+<span class="iconfont">&#x33;</span>
+
+
+

"iconfont" 是你项目下的 font-family。可以通过编辑项目查看,默认是 "iconfont"。

+
+
+
+
+
    + +
  • + +
    + hdfs +
    +
    .icon-hdfs +
    +
  • + +
  • + +
    + hdfs-置灰 +
    +
    .icon-hdfs-zhihui +
    +
  • + +
  • + +
    + 保存 +
    +
    .icon-baocun +
    +
  • + +
  • + +
    + 参数 +
    +
    .icon-canshu +
    +
  • + +
  • + +
    + 执行 +
    +
    .icon-zhihang +
    +
  • + +
  • + +
    + 发布 +
    +
    .icon-fabu +
    +
  • + +
  • + +
    + 资源 +
    +
    .icon-ziyuan +
    +
  • + +
  • + +
    + 蓝色箭头带框 +
    +
    .icon-lansejiantoudaikuang +
    +
  • + +
  • + +
    + 箭头 +
    +
    .icon-jiantou +
    +
  • + +
  • + +
    + hive +
    +
    .icon-hive +
    +
  • + +
  • + +
    + shell +
    +
    .icon-shell1 +
    +
  • + +
  • + +
    + copy +
    +
    .icon-copy +
    +
  • + +
  • + +
    + leftfold +
    +
    .icon-leftfold +
    +
  • + +
  • + +
    + 更多省略号 +
    +
    .icon-gengduoshenglvehao +
    +
  • + +
  • + +
    + unfold +
    +
    .icon-unfold1 +
    +
  • + +
  • + +
    + 收起 +
    +
    .icon-shouqi1 +
    +
  • + +
  • + +
    + 展开 +
    +
    .icon-zhankai +
    +
  • + +
  • + +
    + 固定到导航栏 +
    +
    .icon-gudingdaodaohanglan +
    +
  • + +
  • + +
    + search +
    +
    .icon-search +
    +
  • + +
  • + +
    + spark process +
    +
    .icon-a-sparkprocess +
    +
  • + +
  • + +
    + hive process +
    +
    .icon-a-hiveprocess +
    +
  • + +
  • + +
    + 删除的 hive table +
    +
    .icon-a-shanchudehivetable +
    +
  • + +
  • + +
    + hive table +
    +
    .icon-a-hivetable +
    +
  • + +
  • + +
    + 主题域配置 +
    +
    .icon-zhutiyupeizhi +
    +
  • + +
  • + +
    + 分层配置 +
    +
    .icon-fencengpeizhi +
    +
  • + +
  • + +
    + 已收藏 +
    +
    .icon-yishoucang +
    +
  • + +
  • + +
    + 未收藏 +
    +
    .icon-weishoucang +
    +
  • + +
  • + +
    + star +
    +
    .icon-star +
    +
  • + +
  • + +
    + star-filled +
    +
    .icon-star-filled +
    +
  • + +
  • + +
    + 编辑 +
    +
    .icon-bianji +
    +
  • + +
  • + +
    + 表总数 +
    +
    .icon-biaozongshu +
    +
  • + +
  • + +
    + 数据库总数 +
    +
    .icon-shujukuzongshu +
    +
  • + +
  • + +
    + 总存储量 +
    +
    .icon-zongcunchuliang +
    +
  • + +
  • + +
    + 工作流 +
    +
    .icon-gongzuoliu +
    +
  • + +
  • + +
    + 新增 +
    +
    .icon-xinzeng1 +
    +
  • + +
  • + +
    + 日期 +
    +
    .icon-riqi +
    +
  • + +
  • + +
    + 组件接入管理 +
    +
    .icon-zujianjieruguanli +
    +
  • + +
  • + +
    + 控制台 +
    +
    .icon-kongzhitai +
    +
  • + +
  • + +
    + 部门和用户管理 +
    +
    .icon-bumenheyonghuguanli +
    +
  • + +
  • + +
    + 电信 5G天翼云纯色logo +
    +
    .icon-a-dianxin5Gtianyiyunchunselogo +
    +
  • + +
  • + +
    + 勾 +
    +
    .icon-gou +
    +
  • + +
  • + +
    + 下拉 +
    +
    .icon-xiala +
    +
  • + +
  • + +
    + 服务管理 +
    +
    .icon-fuwuguanli +
    +
  • + +
  • + +
    + 收起 +
    +
    .icon-shouqi +
    +
  • + +
  • + +
    + 总调用次数 +
    +
    .icon-zongtiaoyongcishu +
    +
  • + +
  • + +
    + api +
    +
    .icon-api +
    +
  • + +
  • + +
    + 属性 +
    +
    .icon-shuxing +
    +
  • + +
  • + +
    + API调用 +
    +
    .icon-APItiaoyong +
    +
  • + +
  • + +
    + 总执行时长用量 +
    +
    .icon-zongzhihangshichangyongliang +
    +
  • + +
  • + +
    + API监控 +
    +
    .icon-APIjiankong +
    +
  • + +
  • + +
    + API测试 +
    +
    .icon-APIceshi +
    +
  • + +
  • + +
    + API管理 +
    +
    .icon-APIguanli +
    +
  • + +
  • + +
    + ! +
    +
    .icon-a- +
    +
  • + +
  • + +
    + 版本 +
    +
    .icon-banben +
    +
  • + +
  • + +
    + 测试 +
    +
    .icon-ceshi +
    +
  • + +
  • + +
    + 服务表 +
    +
    .icon-fuwubiao +
    +
  • + +
  • + +
    + 服务开发 +
    +
    .icon-fuwukaifa +
    +
  • + +
  • + +
    + 新增 +
    +
    .icon-xinzeng +
    +
  • + +
  • + +
    + 内存卡底 +
    +
    .icon-neicunkadi +
    +
  • + +
  • + +
    + 矩形 +
    +
    .icon-juxing +
    +
  • + +
  • + +
    + 问号 +
    +
    .icon-wenhao +
    +
  • + +
  • + +
    + 调度 +
    +
    .icon-tiaodu +
    +
  • + +
  • + +
    + 画板 +
    +
    .icon-huaban +
    +
  • + +
  • + +
    + loading +
    +
    .icon-xingzhuang +
    +
  • + +
  • + +
    + file +
    +
    .icon-file +
    +
  • + +
  • + +
    + plus +
    +
    .icon-plus +
    +
  • + +
  • + +
    + unfold +
    +
    .icon-unfold +
    +
  • + +
  • + +
    + put away +
    +
    .icon-putaway +
    +
  • + +
  • + +
    + open +
    +
    .icon-open +
    +
  • + +
  • + +
    + delete +
    +
    .icon-delete +
    +
  • + +
  • + +
    + flow +
    +
    .icon-flow +
    +
  • + +
  • + +
    + close +
    +
    .icon-close +
    +
  • + +
  • + +
    + project +
    +
    .icon-project +
    +
  • + +
  • + +
    + shell +
    +
    .icon-shell +
    +
  • + +
  • + +
    + icon-kill +
    +
    .icon-kill +
    +
  • + +
  • + +
    + icon-ready-stop +
    +
    .icon-ready-stop +
    +
  • + +
  • + +
    + icon-ready-pause +
    +
    .icon-ready-pause +
    +
  • + +
  • + +
    + DELAY_EXECUTION +
    +
    .icon-delay-execution +
    +
  • + +
  • + +
    + NEED_FAULT_TOLERANCE +
    +
    .icon-need-fault-tolerance +
    +
  • + +
  • + +
    + PAUSE +
    +
    .icon-pause +
    +
  • + +
  • + +
    + STOP +
    +
    .icon-stop +
    +
  • + +
  • + +
    + WAITTING_DEPEND +
    +
    .icon-watting-depend +
    +
  • + +
  • + +
    + FORCED_SUCCES +
    +
    .icon-forced-success +
    +
  • + +
  • + +
    + SUCCESS +
    +
    .icon-success +
    +
  • + +
  • + +
    + SUBMITTED_SUCCES +
    +
    .icon-submitted-success +
    +
  • + +
  • + +
    + FAILURE +
    +
    .icon-failure +
    +
  • + +
  • + +
    + RUNNING_EXECUTIO +
    +
    .icon-running-execution +
    +
  • + +
  • + +
    + WAITTING_THREAD +
    +
    .icon-waitting-thread +
    +
  • + +
  • + +
    + REFRESH +
    +
    .icon-refresh +
    +
  • + +
  • + +
    + FULL_SCREEN +
    +
    .icon-full-screen +
    +
  • + +
  • + +
    + CANCLE_FULL_SCREEN +
    +
    .icon-cancel-full-screeen +
    +
  • + +
  • + +
    + DOWNLOAD +
    +
    .icon-download +
    +
  • + +
+
+

font-class 引用

+
+ +

font-class 是 Unicode 使用方式的一种变种,主要是解决 Unicode 书写不直观,语意不明确的问题。

+

与 Unicode 使用方式相比,具有如下特点:

+
    +
  • 相比于 Unicode 语意明确,书写更直观。可以很容易分辨这个 icon 是什么。
  • +
  • 因为使用 class 来定义图标,所以当要替换图标时,只需要修改 class 里面的 Unicode 引用。
  • +
+

使用步骤如下:

+

第一步:引入项目下面生成的 fontclass 代码:

+
<link rel="stylesheet" href="./iconfont.css">
+
+

第二步:挑选相应图标并获取类名,应用于页面:

+
<span class="iconfont icon-xxx"></span>
+
+
+

" + iconfont" 是你项目下的 font-family。可以通过编辑项目查看,默认是 "iconfont"。

+
+
+
+
+
    + +
  • + +
    hdfs
    +
    #icon-hdfs
    +
  • + +
  • + +
    hdfs-置灰
    +
    #icon-hdfs-zhihui
    +
  • + +
  • + +
    保存
    +
    #icon-baocun
    +
  • + +
  • + +
    参数
    +
    #icon-canshu
    +
  • + +
  • + +
    执行
    +
    #icon-zhihang
    +
  • + +
  • + +
    发布
    +
    #icon-fabu
    +
  • + +
  • + +
    资源
    +
    #icon-ziyuan
    +
  • + +
  • + +
    蓝色箭头带框
    +
    #icon-lansejiantoudaikuang
    +
  • + +
  • + +
    箭头
    +
    #icon-jiantou
    +
  • + +
  • + +
    hive
    +
    #icon-hive
    +
  • + +
  • + +
    shell
    +
    #icon-shell1
    +
  • + +
  • + +
    copy
    +
    #icon-copy
    +
  • + +
  • + +
    leftfold
    +
    #icon-leftfold
    +
  • + +
  • + +
    更多省略号
    +
    #icon-gengduoshenglvehao
    +
  • + +
  • + +
    unfold
    +
    #icon-unfold1
    +
  • + +
  • + +
    收起
    +
    #icon-shouqi1
    +
  • + +
  • + +
    展开
    +
    #icon-zhankai
    +
  • + +
  • + +
    固定到导航栏
    +
    #icon-gudingdaodaohanglan
    +
  • + +
  • + +
    search
    +
    #icon-search
    +
  • + +
  • + +
    spark process
    +
    #icon-a-sparkprocess
    +
  • + +
  • + +
    hive process
    +
    #icon-a-hiveprocess
    +
  • + +
  • + +
    删除的 hive table
    +
    #icon-a-shanchudehivetable
    +
  • + +
  • + +
    hive table
    +
    #icon-a-hivetable
    +
  • + +
  • + +
    主题域配置
    +
    #icon-zhutiyupeizhi
    +
  • + +
  • + +
    分层配置
    +
    #icon-fencengpeizhi
    +
  • + +
  • + +
    已收藏
    +
    #icon-yishoucang
    +
  • + +
  • + +
    未收藏
    +
    #icon-weishoucang
    +
  • + +
  • + +
    star
    +
    #icon-star
    +
  • + +
  • + +
    star-filled
    +
    #icon-star-filled
    +
  • + +
  • + +
    编辑
    +
    #icon-bianji
    +
  • + +
  • + +
    表总数
    +
    #icon-biaozongshu
    +
  • + +
  • + +
    数据库总数
    +
    #icon-shujukuzongshu
    +
  • + +
  • + +
    总存储量
    +
    #icon-zongcunchuliang
    +
  • + +
  • + +
    工作流
    +
    #icon-gongzuoliu
    +
  • + +
  • + +
    新增
    +
    #icon-xinzeng1
    +
  • + +
  • + +
    日期
    +
    #icon-riqi
    +
  • + +
  • + +
    组件接入管理
    +
    #icon-zujianjieruguanli
    +
  • + +
  • + +
    控制台
    +
    #icon-kongzhitai
    +
  • + +
  • + +
    部门和用户管理
    +
    #icon-bumenheyonghuguanli
    +
  • + +
  • + +
    电信 5G天翼云纯色logo
    +
    #icon-a-dianxin5Gtianyiyunchunselogo
    +
  • + +
  • + +
    +
    #icon-gou
    +
  • + +
  • + +
    下拉
    +
    #icon-xiala
    +
  • + +
  • + +
    服务管理
    +
    #icon-fuwuguanli
    +
  • + +
  • + +
    收起
    +
    #icon-shouqi
    +
  • + +
  • + +
    总调用次数
    +
    #icon-zongtiaoyongcishu
    +
  • + +
  • + +
    api
    +
    #icon-api
    +
  • + +
  • + +
    属性
    +
    #icon-shuxing
    +
  • + +
  • + +
    API调用
    +
    #icon-APItiaoyong
    +
  • + +
  • + +
    总执行时长用量
    +
    #icon-zongzhihangshichangyongliang
    +
  • + +
  • + +
    API监控
    +
    #icon-APIjiankong
    +
  • + +
  • + +
    API测试
    +
    #icon-APIceshi
    +
  • + +
  • + +
    API管理
    +
    #icon-APIguanli
    +
  • + +
  • + +
    +
    #icon-a-
    +
  • + +
  • + +
    版本
    +
    #icon-banben
    +
  • + +
  • + +
    测试
    +
    #icon-ceshi
    +
  • + +
  • + +
    服务表
    +
    #icon-fuwubiao
    +
  • + +
  • + +
    服务开发
    +
    #icon-fuwukaifa
    +
  • + +
  • + +
    新增
    +
    #icon-xinzeng
    +
  • + +
  • + +
    内存卡底
    +
    #icon-neicunkadi
    +
  • + +
  • + +
    矩形
    +
    #icon-juxing
    +
  • + +
  • + +
    问号
    +
    #icon-wenhao
    +
  • + +
  • + +
    调度
    +
    #icon-tiaodu
    +
  • + +
  • + +
    画板
    +
    #icon-huaban
    +
  • + +
  • + +
    loading
    +
    #icon-xingzhuang
    +
  • + +
  • + +
    file
    +
    #icon-file
    +
  • + +
  • + +
    plus
    +
    #icon-plus
    +
  • + +
  • + +
    unfold
    +
    #icon-unfold
    +
  • + +
  • + +
    put away
    +
    #icon-putaway
    +
  • + +
  • + +
    open
    +
    #icon-open
    +
  • + +
  • + +
    delete
    +
    #icon-delete
    +
  • + +
  • + +
    flow
    +
    #icon-flow
    +
  • + +
  • + +
    close
    +
    #icon-close
    +
  • + +
  • + +
    project
    +
    #icon-project
    +
  • + +
  • + +
    shell
    +
    #icon-shell
    +
  • + +
  • + +
    icon-kill
    +
    #icon-kill
    +
  • + +
  • + +
    icon-ready-stop
    +
    #icon-ready-stop
    +
  • + +
  • + +
    icon-ready-pause
    +
    #icon-ready-pause
    +
  • + +
  • + +
    DELAY_EXECUTION
    +
    #icon-delay-execution
    +
  • + +
  • + +
    NEED_FAULT_TOLERANCE
    +
    #icon-need-fault-tolerance
    +
  • + +
  • + +
    PAUSE
    +
    #icon-pause
    +
  • + +
  • + +
    STOP
    +
    #icon-stop
    +
  • + +
  • + +
    WAITTING_DEPEND
    +
    #icon-watting-depend
    +
  • + +
  • + +
    FORCED_SUCCES
    +
    #icon-forced-success
    +
  • + +
  • + +
    SUCCESS
    +
    #icon-success
    +
  • + +
  • + +
    SUBMITTED_SUCCES
    +
    #icon-submitted-success
    +
  • + +
  • + +
    FAILURE
    +
    #icon-failure
    +
  • + +
  • + +
    RUNNING_EXECUTIO
    +
    #icon-running-execution
    +
  • + +
  • + +
    WAITTING_THREAD
    +
    #icon-waitting-thread
    +
  • + +
  • + +
    REFRESH
    +
    #icon-refresh
    +
  • + +
  • + +
    FULL_SCREEN
    +
    #icon-full-screen
    +
  • + +
  • + +
    CANCLE_FULL_SCREEN
    +
    #icon-cancel-full-screeen
    +
  • + +
  • + +
    DOWNLOAD
    +
    #icon-download
    +
  • + +
+
+

Symbol 引用

+
+ +

这是一种全新的使用方式,应该说这才是未来的主流,也是平台目前推荐的用法。相关介绍可以参考这篇文章 + 这种用法其实是做了一个 SVG 的集合,与另外两种相比具有如下特点:

+
    +
  • 支持多色图标了,不再受单色限制。
  • +
  • 通过一些技巧,支持像字体那样,通过 font-size, color 来调整样式。
  • +
  • 兼容性较差,支持 IE9+,及现代浏览器。
  • +
  • 浏览器渲染 SVG 的性能一般,还不如 png。
  • +
+

使用步骤如下:

+

第一步:引入项目下面生成的 symbol 代码:

+
<script src="./iconfont.js"></script>
+
+

第二步:加入通用 CSS 代码(引入一次就行):

+
<style>
+.icon {
+  width: 1em;
+  height: 1em;
+  vertical-align: -0.15em;
+  fill: currentColor;
+  overflow: hidden;
+}
+</style>
+
+

第三步:挑选相应图标并获取类名,应用于页面:

+
<svg class="icon" aria-hidden="true">
+  <use xlink:href="#icon-xxx"></use>
+</svg>
+
+
+
+ +
+
+ + + diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.css b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.css new file mode 100644 index 000000000..c47e31d93 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.css @@ -0,0 +1,383 @@ +@font-face { + font-family: "iconfont"; /* Project id 2585050 */ + src: url('iconfont.woff2?t=1642471499077') format('woff2'), + url('iconfont.woff?t=1642471499077') format('woff'), + url('iconfont.ttf?t=1642471499077') format('truetype'); +} + +.iconfont { + font-family: "iconfont" !important; + font-size: 16px; + font-style: normal; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.icon-hdfs:before { + content: "\e73a"; +} + +.icon-hdfs-zhihui:before { + content: "\e739"; +} + +.icon-baocun:before { + content: "\e734"; +} + +.icon-canshu:before { + content: "\e735"; +} + +.icon-zhihang:before { + content: "\e736"; +} + +.icon-fabu:before { + content: "\e737"; +} + +.icon-ziyuan:before { + content: "\e738"; +} + +.icon-lansejiantoudaikuang:before { + content: "\e72b"; +} + +.icon-jiantou:before { + content: "\e727"; +} + +.icon-hive:before { + content: "\e722"; +} + +.icon-shell1:before { + content: "\e721"; +} + +.icon-copy:before { + content: "\e720"; +} + +.icon-leftfold:before { + content: "\e71e"; +} + +.icon-gengduoshenglvehao:before { + content: "\e71c"; +} + +.icon-unfold1:before { + content: "\e71d"; +} + +.icon-shouqi1:before { + content: "\e71a"; +} + +.icon-zhankai:before { + content: "\e71b"; +} + +.icon-gudingdaodaohanglan:before { + content: "\e719"; +} + +.icon-search:before { + content: "\e713"; +} + +.icon-a-sparkprocess:before { + content: "\e700"; +} + +.icon-a-hiveprocess:before { + content: "\e701"; +} + +.icon-a-shanchudehivetable:before { + content: "\e702"; +} + +.icon-a-hivetable:before { + content: "\e6ff"; +} + +.icon-zhutiyupeizhi:before { + content: "\e6f9"; +} + +.icon-fencengpeizhi:before { + content: "\e6fa"; +} + +.icon-yishoucang:before { + content: "\e6f7"; +} + +.icon-weishoucang:before { + content: "\e6f8"; +} + +.icon-star:before { + content: "\e6f5"; +} + +.icon-star-filled:before { + content: "\e6f6"; +} + +.icon-bianji:before { + content: "\e6f4"; +} + +.icon-biaozongshu:before { + content: "\e6f1"; +} + +.icon-shujukuzongshu:before { + content: "\e6f2"; +} + +.icon-zongcunchuliang:before { + content: "\e6f3"; +} + +.icon-gongzuoliu:before { + content: "\e6f0"; +} + +.icon-xinzeng1:before { + content: "\e6ef"; +} + +.icon-riqi:before { + content: "\e6ee"; +} + +.icon-zujianjieruguanli:before { + content: "\e6eb"; +} + +.icon-kongzhitai:before { + content: "\e6ec"; +} + +.icon-bumenheyonghuguanli:before { + content: "\e6ed"; +} + +.icon-a-dianxin5Gtianyiyunchunselogo:before { + content: "\e6c7"; +} + +.icon-gou:before { + content: "\e6bb"; +} + +.icon-xiala:before { + content: "\e6bc"; +} + +.icon-fuwuguanli:before { + content: "\e6bd"; +} + +.icon-shouqi:before { + content: "\e6be"; +} + +.icon-zongtiaoyongcishu:before { + content: "\e6bf"; +} + +.icon-api:before { + content: "\e6c0"; +} + +.icon-shuxing:before { + content: "\e6c1"; +} + +.icon-APItiaoyong:before { + content: "\e6c2"; +} + +.icon-zongzhihangshichangyongliang:before { + content: "\e6c3"; +} + +.icon-APIjiankong:before { + content: "\e6c4"; +} + +.icon-APIceshi:before { + content: "\e6c5"; +} + +.icon-APIguanli:before { + content: "\e6c6"; +} + +.icon-a-:before { + content: "\e6b3"; +} + +.icon-banben:before { + content: "\e6b4"; +} + +.icon-ceshi:before { + content: "\e6b5"; +} + +.icon-fuwubiao:before { + content: "\e6b7"; +} + +.icon-fuwukaifa:before { + content: "\e6b9"; +} + +.icon-xinzeng:before { + content: "\e6ba"; +} + +.icon-neicunkadi:before { + content: "\e6b0"; +} + +.icon-juxing:before { + content: "\e6af"; +} + +.icon-wenhao:before { + content: "\e6ac"; +} + +.icon-tiaodu:before { + content: "\e6ad"; +} + +.icon-huaban:before { + content: "\e6ae"; +} + +.icon-xingzhuang:before { + content: "\e84b"; +} + +.icon-file:before { + content: "\e693"; +} + +.icon-plus:before { + content: "\e692"; +} + +.icon-unfold:before { + content: "\e690"; +} + +.icon-putaway:before { + content: "\e691"; +} + +.icon-open:before { + content: "\e68a"; +} + +.icon-delete:before { + content: "\e68c"; +} + +.icon-flow:before { + content: "\e68d"; +} + +.icon-close:before { + content: "\e68e"; +} + +.icon-project:before { + content: "\e68f"; +} + +.icon-shell:before { + content: "\e671"; +} + +.icon-kill:before { + content: "\e667"; +} + +.icon-ready-stop:before { + content: "\e66b"; +} + +.icon-ready-pause:before { + content: "\e66d"; +} + +.icon-delay-execution:before { + content: "\e663"; +} + +.icon-need-fault-tolerance:before { + content: "\e664"; +} + +.icon-pause:before { + content: "\e665"; +} + +.icon-stop:before { + content: "\e666"; +} + +.icon-watting-depend:before { + content: "\e668"; +} + +.icon-forced-success:before { + content: "\e669"; +} + +.icon-success:before { + content: "\e66a"; +} + +.icon-submitted-success:before { + content: "\e66c"; +} + +.icon-failure:before { + content: "\e66e"; +} + +.icon-running-execution:before { + content: "\e66f"; +} + +.icon-waitting-thread:before { + content: "\e670"; +} + +.icon-refresh:before { + content: "\e65f"; +} + +.icon-full-screen:before { + content: "\e660"; +} + +.icon-cancel-full-screeen:before { + content: "\e661"; +} + +.icon-download:before { + content: "\e662"; +} + diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.js b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.js new file mode 100644 index 000000000..e9d11b17d --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.js @@ -0,0 +1 @@ +!function(a){var l,h,c,t,p,i='',v=(v=document.getElementsByTagName("script"))[v.length-1].getAttribute("data-injectcss"),z=function(a,l){l.parentNode.insertBefore(a,l)};if(v&&!a.__iconfont__svg__cssinject__){a.__iconfont__svg__cssinject__=!0;try{document.write("")}catch(a){console&&console.log(a)}}function m(){p||(p=!0,c())}function d(){try{t.documentElement.doScroll("left")}catch(a){return void setTimeout(d,50)}m()}l=function(){var a,l;(l=document.createElement("div")).innerHTML=i,i=null,(a=l.getElementsByTagName("svg")[0])&&(a.setAttribute("aria-hidden","true"),a.style.position="absolute",a.style.width=0,a.style.height=0,a.style.overflow="hidden",l=a,(a=document.body).firstChild?z(l,a.firstChild):a.appendChild(l))},document.addEventListener?~["complete","loaded","interactive"].indexOf(document.readyState)?setTimeout(l,0):(h=function(){document.removeEventListener("DOMContentLoaded",h,!1),l()},document.addEventListener("DOMContentLoaded",h,!1)):document.attachEvent&&(c=l,t=a.document,p=!1,d(),t.onreadystatechange=function(){"complete"==t.readyState&&(t.onreadystatechange=null,m())})}(window); \ No newline at end of file diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.json b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.json new file mode 100644 index 000000000..3c8a371d0 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.json @@ -0,0 +1,653 @@ +{ + "id": "2585050", + "name": "DataSphere Studio", + "font_family": "iconfont", + "css_prefix_text": "icon-", + "description": "", + "glyphs": [ + { + "icon_id": "27222536", + "name": "hdfs", + "font_class": "hdfs", + "unicode": "e73a", + "unicode_decimal": 59194 + }, + { + "icon_id": "27222535", + "name": "hdfs-置灰", + "font_class": "hdfs-zhihui", + "unicode": "e739", + "unicode_decimal": 59193 + }, + { + "icon_id": "27165376", + "name": "保存", + "font_class": "baocun", + "unicode": "e734", + "unicode_decimal": 59188 + }, + { + "icon_id": "27165377", + "name": "参数", + "font_class": "canshu", + "unicode": "e735", + "unicode_decimal": 59189 + }, + { + "icon_id": "27165378", + "name": "执行", + "font_class": "zhihang", + "unicode": "e736", + "unicode_decimal": 59190 + }, + { + "icon_id": "27165379", + "name": "发布", + "font_class": "fabu", + "unicode": "e737", + "unicode_decimal": 59191 + }, + { + "icon_id": "27165380", + "name": "资源", + "font_class": "ziyuan", + "unicode": "e738", + "unicode_decimal": 59192 + }, + { + "icon_id": "26528093", + "name": "蓝色箭头带框", + "font_class": "lansejiantoudaikuang", + "unicode": "e72b", + "unicode_decimal": 59179 + }, + { + "icon_id": "26523520", + "name": "箭头", + "font_class": "jiantou", + "unicode": "e727", + "unicode_decimal": 59175 + }, + { + "icon_id": "26454158", + "name": "hive", + "font_class": "hive", + "unicode": "e722", + "unicode_decimal": 59170 + }, + { + "icon_id": "26454156", + "name": "shell", + "font_class": "shell1", + "unicode": "e721", + "unicode_decimal": 59169 + }, + { + "icon_id": "26440997", + "name": "copy", + "font_class": "copy", + "unicode": "e720", + "unicode_decimal": 59168 + }, + { + "icon_id": "26058470", + "name": "leftfold", + "font_class": "leftfold", + "unicode": "e71e", + "unicode_decimal": 59166 + }, + { + "icon_id": "26055468", + "name": "更多省略号", + "font_class": "gengduoshenglvehao", + "unicode": "e71c", + "unicode_decimal": 59164 + }, + { + "icon_id": "26055469", + "name": "unfold", + "font_class": "unfold1", + "unicode": "e71d", + "unicode_decimal": 59165 + }, + { + "icon_id": "26016697", + "name": "收起", + "font_class": "shouqi1", + "unicode": "e71a", + "unicode_decimal": 59162 + }, + { + "icon_id": "26016698", + "name": "展开", + "font_class": "zhankai", + "unicode": "e71b", + "unicode_decimal": 59163 + }, + { + "icon_id": "25934170", + "name": "固定到导航栏", + "font_class": "gudingdaodaohanglan", + "unicode": "e719", + "unicode_decimal": 59161 + }, + { + "icon_id": "25535513", + "name": "search", + "font_class": "search", + "unicode": "e713", + "unicode_decimal": 59155 + }, + { + "icon_id": "24156532", + "name": "spark process", + "font_class": "a-sparkprocess", + "unicode": "e700", + "unicode_decimal": 59136 + }, + { + "icon_id": "24156533", + "name": "hive process", + "font_class": "a-hiveprocess", + "unicode": "e701", + "unicode_decimal": 59137 + }, + { + "icon_id": "24156534", + "name": "删除的 hive table", + "font_class": "a-shanchudehivetable", + "unicode": "e702", + "unicode_decimal": 59138 + }, + { + "icon_id": "24156421", + "name": "hive table", + "font_class": "a-hivetable", + "unicode": "e6ff", + "unicode_decimal": 59135 + }, + { + "icon_id": "23905974", + "name": "主题域配置", + "font_class": "zhutiyupeizhi", + "unicode": "e6f9", + "unicode_decimal": 59129 + }, + { + "icon_id": "23905975", + "name": "分层配置", + "font_class": "fencengpeizhi", + "unicode": "e6fa", + "unicode_decimal": 59130 + }, + { + "icon_id": "23818087", + "name": "已收藏", + "font_class": "yishoucang", + "unicode": "e6f7", + "unicode_decimal": 59127 + }, + { + "icon_id": "23818088", + "name": "未收藏", + "font_class": "weishoucang", + "unicode": "e6f8", + "unicode_decimal": 59128 + }, + { + "icon_id": "23645384", + "name": "star", + "font_class": "star", + "unicode": "e6f5", + "unicode_decimal": 59125 + }, + { + "icon_id": "23645385", + "name": "star-filled", + "font_class": "star-filled", + "unicode": "e6f6", + "unicode_decimal": 59126 + }, + { + "icon_id": "23628719", + "name": "编辑", + "font_class": "bianji", + "unicode": "e6f4", + "unicode_decimal": 59124 + }, + { + "icon_id": "23594976", + "name": "表总数", + "font_class": "biaozongshu", + "unicode": "e6f1", + "unicode_decimal": 59121 + }, + { + "icon_id": "23594977", + "name": "数据库总数", + "font_class": "shujukuzongshu", + "unicode": "e6f2", + "unicode_decimal": 59122 + }, + { + "icon_id": "23594978", + "name": "总存储量", + "font_class": "zongcunchuliang", + "unicode": "e6f3", + "unicode_decimal": 59123 + }, + { + "icon_id": "23476333", + "name": "工作流", + "font_class": "gongzuoliu", + "unicode": "e6f0", + "unicode_decimal": 59120 + }, + { + "icon_id": "23183738", + "name": "新增", + "font_class": "xinzeng1", + "unicode": "e6ef", + "unicode_decimal": 59119 + }, + { + "icon_id": "23183418", + "name": "日期", + "font_class": "riqi", + "unicode": "e6ee", + "unicode_decimal": 59118 + }, + { + "icon_id": "23181388", + "name": "组件接入管理", + "font_class": "zujianjieruguanli", + "unicode": "e6eb", + "unicode_decimal": 59115 + }, + { + "icon_id": "23181389", + "name": "控制台", + "font_class": "kongzhitai", + "unicode": "e6ec", + "unicode_decimal": 59116 + }, + { + "icon_id": "23181390", + "name": "部门和用户管理", + "font_class": "bumenheyonghuguanli", + "unicode": "e6ed", + "unicode_decimal": 59117 + }, + { + "icon_id": "22920894", + "name": "电信 5G天翼云纯色logo", + "font_class": "a-dianxin5Gtianyiyunchunselogo", + "unicode": "e6c7", + "unicode_decimal": 59079 + }, + { + "icon_id": "22890665", + "name": "勾", + "font_class": "gou", + "unicode": "e6bb", + "unicode_decimal": 59067 + }, + { + "icon_id": "22890666", + "name": "下拉", + "font_class": "xiala", + "unicode": "e6bc", + "unicode_decimal": 59068 + }, + { + "icon_id": "22890667", + "name": "服务管理", + "font_class": "fuwuguanli", + "unicode": "e6bd", + "unicode_decimal": 59069 + }, + { + "icon_id": "22890668", + "name": "收起", + "font_class": "shouqi", + "unicode": "e6be", + "unicode_decimal": 59070 + }, + { + "icon_id": "22890669", + "name": "总调用次数", + "font_class": "zongtiaoyongcishu", + "unicode": "e6bf", + "unicode_decimal": 59071 + }, + { + "icon_id": "22890670", + "name": "api", + "font_class": "api", + "unicode": "e6c0", + "unicode_decimal": 59072 + }, + { + "icon_id": "22890671", + "name": "属性", + "font_class": "shuxing", + "unicode": "e6c1", + "unicode_decimal": 59073 + }, + { + "icon_id": "22890672", + "name": "API调用", + "font_class": "APItiaoyong", + "unicode": "e6c2", + "unicode_decimal": 59074 + }, + { + "icon_id": "22890673", + "name": "总执行时长用量", + "font_class": "zongzhihangshichangyongliang", + "unicode": "e6c3", + "unicode_decimal": 59075 + }, + { + "icon_id": "22890674", + "name": "API监控", + "font_class": "APIjiankong", + "unicode": "e6c4", + "unicode_decimal": 59076 + }, + { + "icon_id": "22890675", + "name": "API测试", + "font_class": "APIceshi", + "unicode": "e6c5", + "unicode_decimal": 59077 + }, + { + "icon_id": "22890676", + "name": "API管理", + "font_class": "APIguanli", + "unicode": "e6c6", + "unicode_decimal": 59078 + }, + { + "icon_id": "22890657", + "name": "!", + "font_class": "a-", + "unicode": "e6b3", + "unicode_decimal": 59059 + }, + { + "icon_id": "22890658", + "name": "版本", + "font_class": "banben", + "unicode": "e6b4", + "unicode_decimal": 59060 + }, + { + "icon_id": "22890659", + "name": "测试", + "font_class": "ceshi", + "unicode": "e6b5", + "unicode_decimal": 59061 + }, + { + "icon_id": "22890661", + "name": "服务表", + "font_class": "fuwubiao", + "unicode": "e6b7", + "unicode_decimal": 59063 + }, + { + "icon_id": "22890663", + "name": "服务开发", + "font_class": "fuwukaifa", + "unicode": "e6b9", + "unicode_decimal": 59065 + }, + { + "icon_id": "22890664", + "name": "新增", + "font_class": "xinzeng", + "unicode": "e6ba", + "unicode_decimal": 59066 + }, + { + "icon_id": "22808932", + "name": "内存卡底", + "font_class": "neicunkadi", + "unicode": "e6b0", + "unicode_decimal": 59056 + }, + { + "icon_id": "22808755", + "name": "矩形", + "font_class": "juxing", + "unicode": "e6af", + "unicode_decimal": 59055 + }, + { + "icon_id": "22779777", + "name": "问号", + "font_class": "wenhao", + "unicode": "e6ac", + "unicode_decimal": 59052 + }, + { + "icon_id": "22779778", + "name": "调度", + "font_class": "tiaodu", + "unicode": "e6ad", + "unicode_decimal": 59053 + }, + { + "icon_id": "22779779", + "name": "画板", + "font_class": "huaban", + "unicode": "e6ae", + "unicode_decimal": 59054 + }, + { + "icon_id": "10610779", + "name": "loading", + "font_class": "xingzhuang", + "unicode": "e84b", + "unicode_decimal": 59467 + }, + { + "icon_id": "22298489", + "name": "file", + "font_class": "file", + "unicode": "e693", + "unicode_decimal": 59027 + }, + { + "icon_id": "22292262", + "name": "plus", + "font_class": "plus", + "unicode": "e692", + "unicode_decimal": 59026 + }, + { + "icon_id": "22285589", + "name": "unfold", + "font_class": "unfold", + "unicode": "e690", + "unicode_decimal": 59024 + }, + { + "icon_id": "22285590", + "name": "put away", + "font_class": "putaway", + "unicode": "e691", + "unicode_decimal": 59025 + }, + { + "icon_id": "22285559", + "name": "open", + "font_class": "open", + "unicode": "e68a", + "unicode_decimal": 59018 + }, + { + "icon_id": "22285561", + "name": "delete", + "font_class": "delete", + "unicode": "e68c", + "unicode_decimal": 59020 + }, + { + "icon_id": "22285562", + "name": "flow", + "font_class": "flow", + "unicode": "e68d", + "unicode_decimal": 59021 + }, + { + "icon_id": "22285563", + "name": "close", + "font_class": "close", + "unicode": "e68e", + "unicode_decimal": 59022 + }, + { + "icon_id": "22285564", + "name": "project", + "font_class": "project", + "unicode": "e68f", + "unicode_decimal": 59023 + }, + { + "icon_id": "21990365", + "name": "shell", + "font_class": "shell", + "unicode": "e671", + "unicode_decimal": 58993 + }, + { + "icon_id": "21974034", + "name": "icon-kill", + "font_class": "kill", + "unicode": "e667", + "unicode_decimal": 58983 + }, + { + "icon_id": "21974035", + "name": "icon-ready-stop", + "font_class": "ready-stop", + "unicode": "e66b", + "unicode_decimal": 58987 + }, + { + "icon_id": "21974036", + "name": "icon-ready-pause", + "font_class": "ready-pause", + "unicode": "e66d", + "unicode_decimal": 58989 + }, + { + "icon_id": "21972598", + "name": "DELAY_EXECUTION", + "font_class": "delay-execution", + "unicode": "e663", + "unicode_decimal": 58979 + }, + { + "icon_id": "21972599", + "name": "NEED_FAULT_TOLERANCE", + "font_class": "need-fault-tolerance", + "unicode": "e664", + "unicode_decimal": 58980 + }, + { + "icon_id": "21972600", + "name": "PAUSE", + "font_class": "pause", + "unicode": "e665", + "unicode_decimal": 58981 + }, + { + "icon_id": "21972601", + "name": "STOP", + "font_class": "stop", + "unicode": "e666", + "unicode_decimal": 58982 + }, + { + "icon_id": "21972603", + "name": "WAITTING_DEPEND", + "font_class": "watting-depend", + "unicode": "e668", + "unicode_decimal": 58984 + }, + { + "icon_id": "21972604", + "name": "FORCED_SUCCES", + "font_class": "forced-success", + "unicode": "e669", + "unicode_decimal": 58985 + }, + { + "icon_id": "21972605", + "name": "SUCCESS", + "font_class": "success", + "unicode": "e66a", + "unicode_decimal": 58986 + }, + { + "icon_id": "21972607", + "name": "SUBMITTED_SUCCES", + "font_class": "submitted-success", + "unicode": "e66c", + "unicode_decimal": 58988 + }, + { + "icon_id": "21972609", + "name": "FAILURE", + "font_class": "failure", + "unicode": "e66e", + "unicode_decimal": 58990 + }, + { + "icon_id": "21972610", + "name": "RUNNING_EXECUTIO", + "font_class": "running-execution", + "unicode": "e66f", + "unicode_decimal": 58991 + }, + { + "icon_id": "21972611", + "name": "WAITTING_THREAD", + "font_class": "waitting-thread", + "unicode": "e670", + "unicode_decimal": 58992 + }, + { + "icon_id": "21972589", + "name": "REFRESH", + "font_class": "refresh", + "unicode": "e65f", + "unicode_decimal": 58975 + }, + { + "icon_id": "21972590", + "name": "FULL_SCREEN", + "font_class": "full-screen", + "unicode": "e660", + "unicode_decimal": 58976 + }, + { + "icon_id": "21972591", + "name": "CANCLE_FULL_SCREEN", + "font_class": "cancel-full-screeen", + "unicode": "e661", + "unicode_decimal": 58977 + }, + { + "icon_id": "21972592", + "name": "DOWNLOAD", + "font_class": "download", + "unicode": "e662", + "unicode_decimal": 58978 + } + ] +} diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.ttf b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.ttf new file mode 100644 index 000000000..8d82dc83b Binary files /dev/null and b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.ttf differ diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.woff b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.woff new file mode 100644 index 000000000..129c1781c Binary files /dev/null and b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.woff differ diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.woff2 b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.woff2 new file mode 100644 index 000000000..1cc2c5dbc Binary files /dev/null and b/web/packages/dataGovernance/view/assetsInfo/components/lineage/symbol/iconfont.woff2 differ diff --git a/web/packages/dataGovernance/view/assetsInfo/components/lineage/theme.scss b/web/packages/dataGovernance/view/assetsInfo/components/lineage/theme.scss new file mode 100644 index 000000000..7694b66b9 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/components/lineage/theme.scss @@ -0,0 +1,27 @@ +@butterfly-theme-color-base:#fff; +@butterfly-normal-font-color-base:#222;; +@butterfly-overlay-font-color-base:#FFF; +@butterfly-primary-color-base:#F66902; +@butterfly-box-border-color-base: #D9D9D9; +@butterfly-box-shadow-base:0 2px 3px 0 rgba(0,112,204,0.06); +@butterfly-box-radius-base:100px; +@butterfly-line-color-base: #BFBFBF; +// 主题背景色(画布背景色) +@butterfly-theme-background-color: @butterfly-theme-color-base; +// (各图形框)普通情况背景色 +@butterfly-theme-color: fade(@butterfly-theme-color-base,80%); +// 主题色 +@butterfly-primary-color: @butterfly-primary-color-base; +// 普通情况下字体颜色 +@butterfly-normal-font-color: @butterfly-normal-font-color-base; +// 主题背景色上的字体颜色及icon颜色 +@butterfly-overlay-font-color: @butterfly-overlay-font-color-base; +// 拖动时的背景色 +@butterfly-box-move-background-color: fade(@butterfly-primary-color, 20%); +// 边框的样式 +@butterfly-box-border: 1px solid @butterfly-box-border-color-base; +@butterfly-box-shadow: @butterfly-box-shadow-base; +@butterfly-box-radius: @butterfly-box-radius-base; +@butterfly-box-node-border-hover: 1px solid @butterfly-primary-color; +// 线条样式 +@butterfly-line-color: @butterfly-line-color-base; diff --git a/web/packages/dataGovernance/view/assetsInfo/index.vue b/web/packages/dataGovernance/view/assetsInfo/index.vue new file mode 100644 index 000000000..2ded73148 --- /dev/null +++ b/web/packages/dataGovernance/view/assetsInfo/index.vue @@ -0,0 +1,596 @@ + + + + + diff --git a/web/packages/dataGovernance/view/assetsSearch/index.vue b/web/packages/dataGovernance/view/assetsSearch/index.vue new file mode 100644 index 000000000..10bde16cc --- /dev/null +++ b/web/packages/dataGovernance/view/assetsSearch/index.vue @@ -0,0 +1,634 @@ + + + diff --git a/web/packages/dataGovernance/view/fieldInfo/index.vue b/web/packages/dataGovernance/view/fieldInfo/index.vue new file mode 100644 index 000000000..fc8925fce --- /dev/null +++ b/web/packages/dataGovernance/view/fieldInfo/index.vue @@ -0,0 +1,343 @@ + + + + + diff --git a/web/packages/dataGovernance/view/governance/index.vue b/web/packages/dataGovernance/view/governance/index.vue new file mode 100644 index 000000000..3223a0309 --- /dev/null +++ b/web/packages/dataGovernance/view/governance/index.vue @@ -0,0 +1,12 @@ + + + diff --git a/web/packages/dataGovernance/view/layered/editModal.vue b/web/packages/dataGovernance/view/layered/editModal.vue new file mode 100644 index 000000000..3b524a881 --- /dev/null +++ b/web/packages/dataGovernance/view/layered/editModal.vue @@ -0,0 +1,239 @@ + + + + + diff --git a/web/packages/dataGovernance/view/layered/index.vue b/web/packages/dataGovernance/view/layered/index.vue new file mode 100644 index 000000000..95848d8ee --- /dev/null +++ b/web/packages/dataGovernance/view/layered/index.vue @@ -0,0 +1,303 @@ + + + + + diff --git a/web/packages/dataGovernance/view/modifier/editModal.vue b/web/packages/dataGovernance/view/modifier/editModal.vue new file mode 100644 index 000000000..53e8af852 --- /dev/null +++ b/web/packages/dataGovernance/view/modifier/editModal.vue @@ -0,0 +1,259 @@ + + + + + diff --git a/web/packages/dataGovernance/view/modifier/index.vue b/web/packages/dataGovernance/view/modifier/index.vue new file mode 100644 index 000000000..ff0195f7f --- /dev/null +++ b/web/packages/dataGovernance/view/modifier/index.vue @@ -0,0 +1,251 @@ + + + + + diff --git a/web/packages/dataGovernance/view/rangeInfo/index.vue b/web/packages/dataGovernance/view/rangeInfo/index.vue new file mode 100644 index 000000000..7fa989732 --- /dev/null +++ b/web/packages/dataGovernance/view/rangeInfo/index.vue @@ -0,0 +1,74 @@ + + + + + diff --git a/web/packages/dataGovernance/view/statPeriod/editModal.vue b/web/packages/dataGovernance/view/statPeriod/editModal.vue new file mode 100644 index 000000000..e8aa92a7f --- /dev/null +++ b/web/packages/dataGovernance/view/statPeriod/editModal.vue @@ -0,0 +1,249 @@ + + + + + diff --git a/web/packages/dataGovernance/view/statPeriod/index.vue b/web/packages/dataGovernance/view/statPeriod/index.vue new file mode 100644 index 000000000..540852148 --- /dev/null +++ b/web/packages/dataGovernance/view/statPeriod/index.vue @@ -0,0 +1,270 @@ + + + + + diff --git a/web/packages/dataGovernance/view/subjectDomain/editModal.vue b/web/packages/dataGovernance/view/subjectDomain/editModal.vue new file mode 100644 index 000000000..d67e89b61 --- /dev/null +++ b/web/packages/dataGovernance/view/subjectDomain/editModal.vue @@ -0,0 +1,211 @@ + + + + + diff --git a/web/packages/dataGovernance/view/subjectDomain/index.vue b/web/packages/dataGovernance/view/subjectDomain/index.vue new file mode 100644 index 000000000..1fd94dc23 --- /dev/null +++ b/web/packages/dataGovernance/view/subjectDomain/index.vue @@ -0,0 +1,228 @@ + + + + + diff --git a/web/packages/dataService/assets/images/develop_nav.svg b/web/packages/dataService/assets/images/develop_nav.svg new file mode 100644 index 000000000..7c6714176 --- /dev/null +++ b/web/packages/dataService/assets/images/develop_nav.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dataService/assets/images/dssLogo.png b/web/packages/dataService/assets/images/dssLogo.png new file mode 100644 index 000000000..2619ce7d1 Binary files /dev/null and b/web/packages/dataService/assets/images/dssLogo.png differ diff --git a/web/packages/dataService/assets/images/property.svg b/web/packages/dataService/assets/images/property.svg new file mode 100644 index 000000000..9310b600d --- /dev/null +++ b/web/packages/dataService/assets/images/property.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dataService/assets/images/release.svg b/web/packages/dataService/assets/images/release.svg new file mode 100644 index 000000000..1ea97e32c --- /dev/null +++ b/web/packages/dataService/assets/images/release.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dataService/assets/images/save.svg b/web/packages/dataService/assets/images/save.svg new file mode 100644 index 000000000..6d859583e --- /dev/null +++ b/web/packages/dataService/assets/images/save.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dataService/assets/images/test.svg b/web/packages/dataService/assets/images/test.svg new file mode 100644 index 000000000..394de10c4 --- /dev/null +++ b/web/packages/dataService/assets/images/test.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dataService/assets/images/version.svg b/web/packages/dataService/assets/images/version.svg new file mode 100644 index 000000000..8f9e980d6 --- /dev/null +++ b/web/packages/dataService/assets/images/version.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dataService/i18n/en.json b/web/packages/dataService/i18n/en.json new file mode 100644 index 000000000..64e5533c8 --- /dev/null +++ b/web/packages/dataService/i18n/en.json @@ -0,0 +1,104 @@ +{ + "message": { + "dataService": { + "dataService": "Data Service", + "serviceManage": "Service Management", + "operation": "operation", + "cancel": "cancel", + "ok": "ok", + "apiIndex": { + "publishApi": "API Published", + "apiName": "api name", + "online": "online", + "offline": "offline", + "test": "test", + "copy": "copy curl", + "copied": "copied", + "copyTips": "get curl success", + "copyBtn": "copy", + "col_apiName": "ApiName", + "col_groupName": "GroupName", + "col_datasourceName": "Datasource", + "col_apiType": "ApiType", + "col_status": "Status", + "onlined": "online", + "offlined": "offline", + "offlineApiTitle": "Are you sure offline the api?", + "offlineApiDesc": "you will not call the api which after offline" + }, + "apiMonitor": { + "staticsScreen": "API Statics Screen", + "staticsDetail": "API Statics Detail", + "apiTotal": "API total", + "published": "published", + "notPublish": "unpublished", + "range_week": "week", + "range_yesterday": "yesterday", + "range_today": "today", + "resource": "resource (last 24 hour)", + "dashboard": "dashboard", + "callCnt": "call count", + "callTime": "call time", + "errorTop10": "error top10", + "callTop10": "call count top10", + "col_rank": "Rank", + "col_apiName": "ApiName", + "col_reqCnt": "TotalCount", + "col_failCnt": "FailCount", + "col_failRate": "FailRate", + "col_totalTime": "TotalTime(ms)", + "col_avgTime": "AvgTime(ms)", + "col_groupName": "GroupName", + "col_label": "Label", + "col_createBy": "CreateBy", + "col_updateTime": "updateTime", + "col_createTime": "createTime", + "viewMonitor": "View Monitor" + }, + "apiCall": { + "apiCall": "Api Call", + "edit": "edit", + "delete": "delete", + "deleteApiCallTitle": "Are you sure delete the api?", + "deleteApiCallDesc": "you will not call the api which after delete", + "addAuthorize": "Add Authorize", + "authForm": { + "modalAuthTile": "API Authorize", + "labelName": "appKey name", + "labelFlow": "authorize flow", + "labelExpire": "authorize expire", + "enterName": "name is required", + "enterNameLength": "The name length should not be greater than", + "enterNameDesc": "Must starts with alphabetic characters, followed by only alphanumeric, underscore and Chinese characters", + "enterFlow": "flow is required", + "enterExpire": "expire is required", + "holderName": "input name", + "holderExpire": "input expire", + "short": "short", + "long": "long" + }, + "col_caller": "appKey", + "col_group": "groupName", + "col_expire": "expire", + "col_token": "appSecret", + "col_updateTime": "updateTime", + "col_createTime": "createTime" + }, + "apiTest": { + "apiTest": "API Test", + "requestParam": "Request Params", + "requestLog": "Request Logs", + "response": "Response", + "name": "name", + "type": "type", + "required": "required", + "value": "value", + "holderInput": "input", + "holderArray": "comma separated values", + "start": "start test", + "api_not_selected": "api not selected", + "req_params_not_value": "request params should be valued" + } + } + } +} diff --git a/web/packages/dataService/i18n/zh.json b/web/packages/dataService/i18n/zh.json new file mode 100644 index 000000000..b851cd3aa --- /dev/null +++ b/web/packages/dataService/i18n/zh.json @@ -0,0 +1,104 @@ +{ + "message": { + "dataService": { + "dataService": "数据服务", + "serviceManage": "服务管理", + "operation": "操作", + "cancel": "取消", + "ok": "确定", + "apiIndex": { + "publishApi": "发布的API", + "apiName": "API名称", + "online": "上线", + "offline": "下线", + "test": "测试", + "copy": "复制调用地址", + "copied": "复制成功", + "copyTips": "调用地址获取成功", + "copyBtn": "复制", + "col_apiName": "API名称", + "col_groupName": "业务流程", + "col_datasourceName": "数据源名称", + "col_apiType": "API类型", + "col_status": "状态", + "onlined": "已上线", + "offlined": "已下线", + "offlineApiTitle": "确认要下线API吗", + "offlineApiDesc": "下线API服务,将影响当前线上业务对API的正常调用" + }, + "apiMonitor": { + "staticsScreen": "API计量大屏", + "staticsDetail": "API计量详情", + "apiTotal": "API总数", + "published": "已发布", + "notPublish": "未发布", + "range_week": "7天", + "range_yesterday": "昨天", + "range_today": "今天", + "resource": "服务资源分配 (最近24小时)", + "dashboard": "总体计量", + "callCnt": "总调用次数", + "callTime": "总调用时长", + "errorTop10": "昨日出错排行Top10", + "callTop10": "昨日调用量排行Top10", + "col_rank": "排名", + "col_apiName": "API名称", + "col_reqCnt": "请求次数", + "col_failCnt": "请求失败次数", + "col_failRate": "请求失败率", + "col_totalTime": "执行时长(ms)", + "col_avgTime": "平均调用时长(ms)", + "col_groupName": "业务流程", + "col_label": "标签", + "col_createBy": "负责人", + "col_updateTime": "更新时间", + "col_createTime": "创建时间", + "viewMonitor": "查看监控图表" + }, + "apiCall": { + "edit": "编辑", + "delete": "删除", + "deleteApiCallTitle": "您确认删除该API调用吗", + "deleteApiCallDesc": "删除后,获得权限调用的用户将不能继续调用该API", + "apiCall": "API调用", + "addAuthorize": "新增授权", + "authForm": { + "modalAuthTile": "API授权", + "labelName": "appkey名称", + "labelFlow": "授权的业务流程", + "labelExpire": "授权有效期", + "enterName": "请输入授权的用户登录名", + "enterNameLength": "名称长度不能大于", + "enterNameDesc": "必须以字母开头,且只支持字母、数字、下划线", + "enterFlow": "请选择授权的业务流程", + "enterExpire": "请选择授权有效期", + "holderName": "请输入名称", + "holderExpire": "请选择日期", + "short": "短期", + "long": "长期" + }, + "col_caller": "appKey", + "col_group": "业务流程", + "col_expire": "有效期", + "col_token": "appSecret", + "col_updateTime": "更新时间", + "col_createTime": "创建时间" + }, + "apiTest": { + "apiTest": "API测试", + "requestParam": "请求参数", + "requestLog": "请求详情", + "response": "返回内容", + "name": "参数名称", + "type": "参数类型", + "required": "是否必填", + "value": "值", + "holderInput": "请输入", + "holderArray": "多值以逗号分隔", + "start": "开始测试", + "api_not_selected": "请先选择API", + "req_params_not_value": "请求参数必须赋值" + } + } + } +} diff --git a/web/packages/dataService/module/common/manageMenu.vue b/web/packages/dataService/module/common/manageMenu.vue new file mode 100644 index 000000000..7b492c23b --- /dev/null +++ b/web/packages/dataService/module/common/manageMenu.vue @@ -0,0 +1,76 @@ + + + diff --git a/web/packages/dataService/module/common/navMenu.vue b/web/packages/dataService/module/common/navMenu.vue new file mode 100644 index 000000000..2fccb3688 --- /dev/null +++ b/web/packages/dataService/module/common/navMenu.vue @@ -0,0 +1,176 @@ + + + diff --git a/web/packages/dataService/module/common/rangeGroup.vue b/web/packages/dataService/module/common/rangeGroup.vue new file mode 100644 index 000000000..ba91fef71 --- /dev/null +++ b/web/packages/dataService/module/common/rangeGroup.vue @@ -0,0 +1,138 @@ + + + \ No newline at end of file diff --git a/web/packages/dataService/module/common/test.vue b/web/packages/dataService/module/common/test.vue new file mode 100644 index 000000000..c1f998a87 --- /dev/null +++ b/web/packages/dataService/module/common/test.vue @@ -0,0 +1,282 @@ + + + diff --git a/web/packages/dataService/module/common/tree-item.vue b/web/packages/dataService/module/common/tree-item.vue new file mode 100644 index 000000000..712e99980 --- /dev/null +++ b/web/packages/dataService/module/common/tree-item.vue @@ -0,0 +1,191 @@ + + + diff --git a/web/packages/dataService/module/common/tree.vue b/web/packages/dataService/module/common/tree.vue new file mode 100644 index 000000000..3aa26a4e7 --- /dev/null +++ b/web/packages/dataService/module/common/tree.vue @@ -0,0 +1,116 @@ + + diff --git a/web/packages/dataService/module/common/treeMenu.vue b/web/packages/dataService/module/common/treeMenu.vue new file mode 100644 index 000000000..02980f375 --- /dev/null +++ b/web/packages/dataService/module/common/treeMenu.vue @@ -0,0 +1,239 @@ + + + diff --git a/web/packages/dataService/module/common/util.js b/web/packages/dataService/module/common/util.js new file mode 100644 index 000000000..482ca5f8c --- /dev/null +++ b/web/packages/dataService/module/common/util.js @@ -0,0 +1,12 @@ +/*eslint-disable */ +function dateFormat(date, suffix = '00:00:00') { + const dt = date ? date : new Date(); + const format = [ + dt.getFullYear(), dt.getMonth() + 1, dt.getDate() + ].join('-').replace(/(?=\b\d\b)/g, '0'); // 正则补零 + return `${format} ${suffix}`; +} + +export default { + dateFormat +} \ No newline at end of file diff --git a/web/packages/dataService/module/dataManagement/apiCall.vue b/web/packages/dataService/module/dataManagement/apiCall.vue new file mode 100644 index 000000000..3b1e64938 --- /dev/null +++ b/web/packages/dataService/module/dataManagement/apiCall.vue @@ -0,0 +1,404 @@ + + + + diff --git a/web/packages/dataService/module/dataManagement/apiIndex.vue b/web/packages/dataService/module/dataManagement/apiIndex.vue new file mode 100644 index 000000000..fce07d229 --- /dev/null +++ b/web/packages/dataService/module/dataManagement/apiIndex.vue @@ -0,0 +1,269 @@ + + + + diff --git a/web/packages/dataService/module/dataManagement/apiMonitor.vue b/web/packages/dataService/module/dataManagement/apiMonitor.vue new file mode 100644 index 000000000..b9e7ea0ef --- /dev/null +++ b/web/packages/dataService/module/dataManagement/apiMonitor.vue @@ -0,0 +1,551 @@ + + + diff --git a/web/packages/dataService/module/dataManagement/apiTest.vue b/web/packages/dataService/module/dataManagement/apiTest.vue new file mode 100644 index 000000000..56811ed16 --- /dev/null +++ b/web/packages/dataService/module/dataManagement/apiTest.vue @@ -0,0 +1,11 @@ + + \ No newline at end of file diff --git a/web/packages/dataService/module/dataManagement/index.js b/web/packages/dataService/module/dataManagement/index.js new file mode 100644 index 000000000..d61b0e054 --- /dev/null +++ b/web/packages/dataService/module/dataManagement/index.js @@ -0,0 +1,10 @@ +export default { + name: 'dataMangement', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `${location.protocol}//${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/packages/dataService/module/dataManagement/index.vue b/web/packages/dataService/module/dataManagement/index.vue new file mode 100644 index 000000000..29373538f --- /dev/null +++ b/web/packages/dataService/module/dataManagement/index.vue @@ -0,0 +1,53 @@ + + + + diff --git a/web/packages/dataService/module/dataManagement/monitorChart.vue b/web/packages/dataService/module/dataManagement/monitorChart.vue new file mode 100644 index 000000000..7e92c251f --- /dev/null +++ b/web/packages/dataService/module/dataManagement/monitorChart.vue @@ -0,0 +1,290 @@ + + + \ No newline at end of file diff --git a/web/packages/dataService/module/dataService/apiConfig.vue b/web/packages/dataService/module/dataService/apiConfig.vue new file mode 100644 index 000000000..adc8c1afe --- /dev/null +++ b/web/packages/dataService/module/dataService/apiConfig.vue @@ -0,0 +1,158 @@ + + + + diff --git a/web/packages/dataService/module/dataService/emptyGuide.vue b/web/packages/dataService/module/dataService/emptyGuide.vue new file mode 100644 index 000000000..96e539672 --- /dev/null +++ b/web/packages/dataService/module/dataService/emptyGuide.vue @@ -0,0 +1,86 @@ + + + + diff --git a/web/packages/dataService/module/dataService/index.js b/web/packages/dataService/module/dataService/index.js new file mode 100644 index 000000000..dd5f4d242 --- /dev/null +++ b/web/packages/dataService/module/dataService/index.js @@ -0,0 +1,10 @@ +export default { + name: 'dataService', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `${location.protocol}//${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/packages/dataService/module/dataService/index.vue b/web/packages/dataService/module/dataService/index.vue new file mode 100644 index 000000000..33965bce1 --- /dev/null +++ b/web/packages/dataService/module/dataService/index.vue @@ -0,0 +1,577 @@ + + + + diff --git a/web/packages/dataService/module/dataService/paramsConfig.vue b/web/packages/dataService/module/dataService/paramsConfig.vue new file mode 100644 index 000000000..fdd9e2112 --- /dev/null +++ b/web/packages/dataService/module/dataService/paramsConfig.vue @@ -0,0 +1,1198 @@ + + + + diff --git a/web/packages/dataService/module/header/index.js b/web/packages/dataService/module/header/index.js new file mode 100644 index 000000000..4e0a78d62 --- /dev/null +++ b/web/packages/dataService/module/header/index.js @@ -0,0 +1,5 @@ +export default { + name: 'scriptisHeader', + component: () => import('./index.vue'), + dispatchs: [], +}; \ No newline at end of file diff --git a/web/packages/dataService/module/header/index.scss b/web/packages/dataService/module/header/index.scss new file mode 100644 index 000000000..fd86b1b0a --- /dev/null +++ b/web/packages/dataService/module/header/index.scss @@ -0,0 +1,21 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +@import '@dataspherestudio/shared/common/style/variables.scss'; + +@import '@dataspherestudio/shared/common/style/headerCommon.scss'; + diff --git a/web/packages/dataService/module/header/index.vue b/web/packages/dataService/module/header/index.vue new file mode 100644 index 000000000..e8bcbb86c --- /dev/null +++ b/web/packages/dataService/module/header/index.vue @@ -0,0 +1,101 @@ + + + diff --git a/web/packages/dataService/module/header/userMenu.vue b/web/packages/dataService/module/header/userMenu.vue new file mode 100644 index 000000000..38e34ca8b --- /dev/null +++ b/web/packages/dataService/module/header/userMenu.vue @@ -0,0 +1,109 @@ + + + diff --git a/web/packages/dataService/package.json b/web/packages/dataService/package.json new file mode 100644 index 000000000..bea0b997e --- /dev/null +++ b/web/packages/dataService/package.json @@ -0,0 +1,7 @@ +{ + "name": "@dataspherestudio/dataService", + "version": "1.1.4", + "dependencies": { + "@dataspherestudio/shared": "^1.1.4" + } +} diff --git a/web/packages/dataService/router.js b/web/packages/dataService/router.js new file mode 100644 index 000000000..127f88b2d --- /dev/null +++ b/web/packages/dataService/router.js @@ -0,0 +1,66 @@ +const routes = [ + { + path: '/dataService', + name: 'dataService', + meta: { + publicPage: true + }, + component: () => import('./view/develop/index.vue') + }, + { + path: '/dataManagement', + name: 'dataManagement', + redirect: '/dataManagement/index', + meta: { + publicPage: true + }, + component: () => import('./view/management/index.vue'), + children: [{ + path: 'index', + name: 'dataManagement/index', + meta: { + title: 'API管理', + publicPage: true + }, + component: () => import('./module/dataManagement/apiIndex.vue'), + }, + { + path: 'monitor', + name: 'dataManagement/monitor', + meta: { + title: 'API监控', + publicPage: true + }, + component: () => import('./module/dataManagement/apiMonitor.vue'), + }, + { + path: 'test', + name: 'dataManagement/test', + meta: { + title: 'API测试', + publicPage: true + }, + component: () => import('./module/dataManagement/apiTest.vue'), + }, + { + path: 'test/:apiId', + name: 'dataManagement/test/apiId', + meta: { + title: 'API测试', + publicPage: true + }, + component: () => import('./module/dataManagement/apiTest.vue'), + }, + { + path: 'call', + name: 'dataManagement/call', + meta: { + title: 'API调用', + publicPage: true + }, + component: () => import('./module/dataManagement/apiCall.vue'), + }] + }, +] + +export default routes; diff --git a/web/packages/dataService/view/develop/index.vue b/web/packages/dataService/view/develop/index.vue new file mode 100644 index 000000000..fe6690cba --- /dev/null +++ b/web/packages/dataService/view/develop/index.vue @@ -0,0 +1,12 @@ + + + \ No newline at end of file diff --git a/web/packages/dataService/view/management/index.vue b/web/packages/dataService/view/management/index.vue new file mode 100644 index 000000000..fadd62d15 --- /dev/null +++ b/web/packages/dataService/view/management/index.vue @@ -0,0 +1,13 @@ + + + \ No newline at end of file diff --git a/web/packages/dolphinScheduler/assets/styles/workflow.scss b/web/packages/dolphinScheduler/assets/styles/workflow.scss new file mode 100644 index 000000000..1877b8c03 --- /dev/null +++ b/web/packages/dolphinScheduler/assets/styles/workflow.scss @@ -0,0 +1,268 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +@import "@dataspherestudio/shared/common/style/variables.scss"; +.workflow-wrap { + width: 100%; + height: 100%; + @include bg-color($workspace-background, $dark-workspace-background); + .workflow-nav-tree { + display: flex; + position: fixed; + left: 0; + top: 54px; + bottom: 0; + width: 250px; + @include bg-color($workflow-body-bg-color, $dark-workflow-body-bg-color); + transition: all 0.3s; + &-switch { + height: 40px; + width: 40px; + display: flex; + justify-content: center; + align-items: center; + border-right: 1px solid; + border-color: #dee4ec; + box-sizing: border-box; + } + &.tree-fold { + width: 40px; + height: 40px; + z-index: 1; + .project-nav-tree { + transform: translateX(-304px); + } + } + .project-nav-menu { + z-index: 1; + width: 54px; + // background: #F8F9FC; + @include bg-color(#f8f9fc, $dark-menu-base-color); + border-right: 1px solid #dee4ec; + @include border-color($border-color-base, $dark-active-menu-item); + &-item { + height: 44px; + line-height: 44px; + text-align: center; + cursor: pointer; + @include font-color(#333, $dark-workflow-font-color); + &:hover { + @include bg-color($hover-color-base, $dark-hover-color-base); + } + .nav-icon { + font-size: 26px; + } + } + .active { + @include bg-color($active-menu-item, $dark-active-menu-item); + border-left: 3px solid #2e92f7; + @include border-color($primary-color, $dark-primary-color); + } + } + .project-nav-tree-bottom { + width: 100%; + height: 34px; + padding: 0 15px; + margin-top: 8px; + line-height: 34px; + .project-nav-add { + line-height: 34px; + text-align: center; + height: 34px; + border: 1px dashed #dee4ec; + + font-size: 14px; + color: rgba(0, 0, 0, 0.65); + font-weight: 400; + cursor: pointer; + &:hover { + background-color: #2E92F7; + color: #fff; + } + } + } + .project-nav-tree { + position: absolute; + width: 250px; + left: 0; + top: 0; + bottom: 0; + transition: all 0.3s; + padding: 10px 0; + overflow-y: auto; + border-right: 1px solid #dee4ec; + @include border-color($border-color-base, $dark-menu-base-color); + @include bg-color($light-base-color, $dark-workspace-background); + overflow-x: hidden; + &-top { + display: flex; + // height: 65px; + width: 250px; + padding: 0 12px; + flex-direction: column; + &-t { + display: flex; + height: 25px; + justify-content: space-between; + align-items: center; + &-txt { + + font-size: 14px; + color: rgba(0, 0, 0, 0.65); + text-align: right; + font-weight: 600; + } + &-icon { + cursor: pointer; + font-size: 16px; + } + } + &-b { + height: 40px; + display: flex; + align-items: center; + &-input { + border: none; + border-bottom: 1px solid #dee4ec; + display: inline-block; + height: 24px; + width: 208px; + margin-left: 8px; + &::placeholder { + color: rgba(0, 0, 0, 0.25); + } + } + } + } + } + } + .streamisContainer { + display: flex; + min-height: 100%; + } + .workflowListContainer { + display: flex; + min-height: 100%; + padding: $padding-25; + border: $border-width-base $border-style-base $border-color-base; + @include border-color($border-color-base, $dark-workspace-body-bg-color); + @include bg-color($workspace-body-bg-color, $dark-workspace-body-bg-color); + .workflowListLeft { + position: relative; + flex: 1; + .tabs-content { + /deep/ .ivu-tabs-tab { + font-size: $font-size-large; + } + } + } + } + .workflow-tabs { + width: 100%; + height: 32px; + margin: 4px 0; + white-space: nowrap; + line-height: $line-height-base; + font-size: $font-size-base; + position: relative; + color: $text-color; + border-bottom: $border-width-base $border-style-base $border-color-base; + @include border-color($border-color-base, $dark-border-color-base); + padding: 0 5px; + .workflow-tabs-item { + margin: 0; + height: 31px; + padding: 5px 16px 4px; + border-bottom: $border-width-base $border-style-base $border-color-base; + @include border-color($border-color-base, $dark-border-color-base); + border-radius: 4px 4px 0 0; + // background: #f8f8f9; + @include bg-color($workflow-body-bg-color, $dark-workflow-body-bg-color); + display: inline-block; + cursor: pointer; + position: relative; + &.active { + height: 32px; + padding-bottom: 5px; + background: #fff; + transform: translateZ(0); + border: 1px solid #dcdee2; + border-bottom: 1px solid #fff; + // color: #2d8cf0; + @include font-color($primary-color, $dark-primary-color); + } + .workflow-tabs-name { + display: inline-block; + } + .workflow-tabs-close { + width: 22px; + margin-right: -6px; + height: 22px; + font-size: 22px; + color: #999; + text-align: right; + vertical-align: middle; + overflow: hidden; + position: relative; + color: $text-color; + border-bottom: $border-width-base $border-style-base $border-color-base; + @include border-color($border-color-base, $dark-border-color-base); + padding: 0 5px; + .workflow-tabs-item { + margin: 0; + height: 31px; + padding: 5px 16px 4px; + border-bottom: $border-width-base $border-style-base $border-color-base; + @include border-color($border-color-base, $dark-border-color-base); + border-radius: 4px 4px 0 0; + // background: #f8f8f9; + @include bg-color($workflow-body-bg-color, $dark-workflow-body-bg-color); + display: inline-block; + cursor: pointer; + position: relative; + &.active { + height: 32px; + padding-bottom: 5px; + background: #fff; + transform: translateZ(0); + border: 1px solid #dcdee2; + border-bottom:1px solid #fff; + // color: #2d8cf0; + @include font-color($primary-color, $dark-primary-color); + } + .workflow-tabs-name { + display: inline-block; + } + .workflow-tabs-close { + width: 22px; + margin-right: -6px; + height: 22px; + font-size: 22px; + color: #999; + text-align: right; + vertical-align: middle; + overflow: hidden; + position: relative; + top: -1px; + transform-origin: 100% 50%; + transition: all .3s ease-in-out; + cursor: pointer; + } + } + } + } + } +} diff --git a/web/packages/dolphinScheduler/components/tabList/index.scss b/web/packages/dolphinScheduler/components/tabList/index.scss new file mode 100644 index 000000000..390a5676e --- /dev/null +++ b/web/packages/dolphinScheduler/components/tabList/index.scss @@ -0,0 +1,173 @@ +@import "@dataspherestudio/shared/common/style/variables.scss"; +.workflowTabContainer { + margin-left: 250px; + transition: margin-left 0.3s; + position: relative; + height: 100%; + box-sizing: border-box; + display: flex; + flex-direction: column; + &.tree-fold { + margin-left: 0px; + } + .tap-menu { + min-width: 98px; + height: 40px; + box-sizing: border-box; + line-height: 40px; + text-align: center; + padding: 0 12px; + } + + .tap-bar { + &.tree-fold { + margin-left: 40px; + } + @include bg-color(#f8f9fc, $dark-workspace-body-bg-color); + .topTabList { + padding: 25px $padding-25 0px; + flex: none; + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + .topLeft { + flex: 1; + min-width: 320px; + .ivu-breadcrumb { + font-size: 21px; + /deep/.ivu-breadcrumb-item-separator { + // color: rgba(0,0,0,0.65); + @include font-color($light-text-color, $dark-text-color); + } + } + } + .buttonChange { + flex: none; + .ivu-btn { + font-size: 14px; + } + .bottomTapList { + padding: 0px $padding-25; + border-bottom: $border-width-base $border-style-base $border-color-base; + @include border-color($background-color-base, $dark-workspace-body-bg-color); + @include font-color($workspace-title-color, $dark-workspace-title-color); + margin-top: 12px; + flex: none; + display: flex; + align-items: center; + font-size: $font-size-large; + .bottomLeftText { + cursor: pointer; + flex: none; + font-size: $font-size-large; + padding: 0 15px; + margin-bottom: -1px; + line-height: 40px; + position: relative; + &::after { + content: ""; + border-left: 1px solid #DEE4EC; + @include border-color($border-color-base, $dark-border-color-base); + width: 0; + position: absolute; + right: -15px; + top: 12px; + height: 16px; + margin: 0 15px; + } + } + .active { + border-bottom: 2px solid $primary-color; + @include border-color($primary-color, $dark-primary-color); + } + .bottomRightContainer { + flex: 1; + height: 40px; + } + } + } + } + .bottomTapList { + border-bottom: $border-width-base $border-style-base $border-color-base; + @include border-color( + $background-color-base, + $dark-workspace-body-bg-color + ); + @include font-color($workspace-title-color, $dark-workspace-title-color); + // margin-top: 12px; + flex: none; + display: flex; + align-items: center; + font-size: 14px; + .bottomLeftText { + cursor: pointer; + flex: none; + font-size: $font-size-large; + padding: 0 15px; + margin-bottom: -1px; + line-height: 40px; + position: relative; + &::after { + content: ""; + border-left: 1px solid #dee4ec; + @include border-color($border-color-base, $dark-border-color-base); + width: 0; + position: absolute; + right: -15px; + top: 12px; + height: 16px; + margin: 0 15px; + } + } + .active { + // border-bottom: 2px solid $primary-color; + // @include border-color($primary-color, $dark-primary-color); + } + .bottomRightContainer { + flex: 1; + height: 40px; + display: flex; + align-items: center; + } + } + } + .defaultSlot { + flex: 1; + } +} + +.tab-item { + display: inline-block; + height: 24px; + line-height: 24px; + // color: $title-color; + @include font-color($workspace-title-color, $dark-workspace-title-color); + cursor: pointer; + min-width: 90px; + max-width: 200px; + padding: 0 10px; + overflow: hidden; + margin-right: 8px; + @include bg-color( #E1E5EA, $dark-workspace-body-bg-color); + border-radius: 12px; + &.active { + height: 24px; + @include font-color($primary-color, $dark-primary-color); + line-height: 24px; + @include bg-color(#E8EEF4, $dark-workspace-body-bg-color); + border-radius: 12px; + } + &:hover { + height: 24px; + @include font-color($primary-color, $dark-primary-color); + line-height: 24px; + border-radius: 12px; + @include bg-color(#D1D7DD, $dark-workspace-body-bg-color); + } +} + +.top-tap-lists { + display: flex; + align-items: center; +} diff --git a/web/packages/dolphinScheduler/components/tabList/index.vue b/web/packages/dolphinScheduler/components/tabList/index.vue new file mode 100644 index 000000000..9cd7d2b14 --- /dev/null +++ b/web/packages/dolphinScheduler/components/tabList/index.vue @@ -0,0 +1,122 @@ + + + diff --git a/web/packages/dolphinScheduler/components/tabList/tabs.vue b/web/packages/dolphinScheduler/components/tabList/tabs.vue new file mode 100644 index 000000000..a9fd6bacd --- /dev/null +++ b/web/packages/dolphinScheduler/components/tabList/tabs.vue @@ -0,0 +1,77 @@ + + + diff --git a/web/packages/dolphinScheduler/i18n/en.json b/web/packages/dolphinScheduler/i18n/en.json new file mode 100644 index 000000000..42aa80f45 --- /dev/null +++ b/web/packages/dolphinScheduler/i18n/en.json @@ -0,0 +1,148 @@ +{ + "message": { + "scheduler": { + "selectState": "Select State", + "taskStatusStatistics": "Task status statistics", + "processStatusStatistics": "Process Status Statistics", + "processDefinitionStatistics": "Process Definition Statistics", + "projectHome": "Project Home", + "home": "Dashboard", + "dashboard": "Operation Dashboard", + "processDefinition": "Process Definition", + "processInstance": "Process Instance", + "taskInstance": "Task Instance", + "run": "run", + "setTime": "timing", + "setTimeManage": "Cron Manage", + "offline": "offline", + "online": "online", + "rerun": "rerun", + "delete": "Delete", + "DELETE": "Delete", + "Unpublished": "Unpublished", + "START_PROCESS": "Start Process", + "START_CURRENT_TASK_PROCESS": "Execute from the current node", + "RECOVER_TOLERANCE_FAULT_PROCESS": "Recover tolerance fault process", + "RECOVER_SUSPENDED_PROCESS": "Resume the suspension process", + "START_FAILURE_TASK_PROCESS": "Execute from the failed nodes", + "COMPLEMENT_DATA": "Complement Data", + "SCHEDULER": "Scheduling execution", + "RECOVER_WAITTING_THREAD": "Recovery waiting thread", + "PAUSE": "Pause", + "STOP": "Stop", + "noUrl": "Unable to download without proper url", + "noMoreLogs": "No more logs", + "updateSuccess": "Update log success", + "noLog": "No log", + "noData": "No Data", + "logLoading": "Loading Log...", + "Loading": "Loading...", + "viewLog": "View log", + "downloadLog": "Download Log", + "refreshLog": "Refresh Log", + "fullScreen": "Enter full screen", + "cancelScreen": "Cancel full screen", + "close": "Close", + "cancel": "Cancel", + "ok": "OK", + "recovery": "Recovery Failed", + "recoverySuspend": "Recovery Suspend", + "copy": "Copy Workflow", + "export": "Export", + "Gantt": "Gantt", + "taskStatus": "Task Status", + "runTask": { + "startTitle": "Please set the parameters before starting", + "timingTitle": "Set parameters before timing", + "processName": "Process Name", + "failureStrategy": "Failure Strategy", + "continue": "Continue", + "end": "End", + "nodeExecution": "Node execution", + "backwardExecution": "Backward execution", + "forwardExecution": "Forward execution", + "executeOnlyTheCurrentNode": "Execute only the current node", + "notificationStrategy": "Notification strategy", + "processPriority": "Process priority", + "workerGroup": "Worker group", + "notificationGroup": "Notification group", + "alarmGroup": "Alarm group", + "complementData": "Complement Data", + "complementProcess?": "Whether it is a complement process?", + "modeOfExecution": "Mode of execution", + "serialExecution": "Serial execution", + "parallelExecution": "Parallel execution", + "scheduleDate": "Schedule date", + "startupParameter": "Startup parameter", + "cancel": "Cancel", + "start": "Start", + "none_1": "none", + "success_1": "success", + "failure_1": "failure", + "All_1": "All", + "receivers": "receivers", + "receiverCcs": "receiverCcs", + "startDate": "startDate", + "endDate": "endDate", + "StartAndStopTime": "Start and stop time", + "Timing": "Timing", + "ExecuteTime": "Execute time", + "Timezone": "Timezone", + "fiveTimes": "Next five execution times", + "create": "Create", + "edit": "Edit", + "inputEmail": "Please enter Email, separate by comma", + "selectTime": "Please select time", + "startNotSameAsEnd": "The start time must not be the same as the end", + "enterCrontab": "Please enter crontab", + "success": "Success", + "operationSuccess": "Operation Success", + "PleaseGoOnline": "Please go online", + "noData": "There is no data for this period of time" + }, + "tasksState": { + "SUBMITTED_SUCCESS": "Submitted successfully", + "RUNNING_EXECUTION": "Executing", + "READY_PAUSE": "Ready to pause", + "PAUSE": "Pause", + "READY_STOP": "Ready to stop", + "STOP": "Stop", + "FAILURE": "Failed", + "SUCCESS": "Success", + "NEED_FAULT_TOLERANCE": "Need fault tolerance", + "KILL": "Kill", + "WAITTING_THREAD": "Waiting for thread", + "WAITTING_DEPEND": "Waiting for dependence", + "DELAY_EXECUTION": "Delay execution", + "FORCED_SUCCESS": "Forced success" + }, + "header": { + "id": "#", + "ProcessName": "Process Name", + "Name": "Name", + "ProcessInstance": "Process Instance", + "nodeType": "Node Type", + "Number": "Number", + "State": "State", + "RunType": "Run Type", + "SchedulingTime": "Scheduling Time", + "SubmitTime": "Submit Time", + "StartTime": "Start Time", + "EndTime": "End Time", + "Duration": "Duration", + "RunTimes": "Run Times", + "retryTimes": "Retry Times", + "FaultTolerantSign": "fault-tolerant sign", + "Executor": "Executor", + "Operation": "Operation", + "CreateTime": "Create Time", + "UpdateTime": "Update Time", + "Description": "Description", + "ModifyBy": "Modify User", + "TimingState": "Timing state", + "crontab": "crontab", + "failureStrategy": "Failure Strategy" + } + } + } +} diff --git a/web/packages/dolphinScheduler/i18n/zh.json b/web/packages/dolphinScheduler/i18n/zh.json new file mode 100644 index 000000000..3e64d910b --- /dev/null +++ b/web/packages/dolphinScheduler/i18n/zh.json @@ -0,0 +1,148 @@ +{ + "message": { + "scheduler": { + "selectState": "选择状态", + "taskStatusStatistics": "任务状态统计", + "processStatusStatistics": "流程状态统计", + "processDefinitionStatistics": "流程定义统计", + "projectHome": "运维大屏", + "home": "运维大屏", + "dashboard": "运维大屏", + "processDefinition": "工作流定义", + "processInstance": "工作流实例", + "taskInstance": "任务实例", + "run": "运行", + "setTime": "定时", + "setTimeManage": "定时管理", + "offline": "下线", + "online": "上线", + "rerun": "重跑", + "delete": "确定删除吗", + "DELETE": "删除", + "Unpublished": "未发布", + "START_PROCESS": "启动工作流", + "START_CURRENT_TASK_PROCESS": "从当前节点开始执行", + "RECOVER_TOLERANCE_FAULT_PROCESS": "恢复被容错的工作流", + "RECOVER_SUSPENDED_PROCESS": "恢复运行流程", + "START_FAILURE_TASK_PROCESS": "从失败节点开始执行", + "COMPLEMENT_DATA": "补数", + "SCHEDULER": "调度执行", + "RECOVER_WAITTING_THREAD": "恢复等待线程", + "PAUSE": "暂停", + "STOP": "停止", + "noUrl": "无下载url无法下载", + "noMoreLogs": "没有更多日志", + "updateSuccess": "日志更新成功", + "noLog": "暂无日志", + "noData": "暂无数据", + "logLoading": "日志加载中...", + "Loading": "加载中...", + "viewLog": "查看日志", + "downloadLog": "下载日志", + "refreshLog": "刷新日志", + "fullScreen": "全屏", + "cancelScreen": "退出全屏", + "close": "关闭", + "cancel": "取消", + "ok": "确定", + "recovery": "恢复失败", + "recoverySuspend": "恢复运行", + "copy": "复制工作流", + "export": "导出", + "Gantt": "甘特图", + "taskStatus": "状态任务", + "runTask": { + "startTitle": "启动前请先设置参数", + "timingTitle": "定时前请先设置参数", + "processName": "工作流名称", + "failureStrategy": "失败策略", + "continue": "继续", + "end": "结束", + "nodeExecution": "节点执行", + "backwardExecution": "向后执行", + "forwardExecution": "向前执行", + "executeOnlyTheCurrentNode": "仅执行当前节点", + "notificationStrategy": "通知策略", + "processPriority": "流程优先级", + "workerGroup": "Worker分组", + "notificationGroup": "通知组", + "alarmGroup": "告警组", + "complementData": "补数", + "complementProcess?": "是否补数?", + "modeOfExecution": "执行方式", + "serialExecution": "串行执行", + "parallelExecution": "并行执行", + "scheduleDate": "调度日期", + "startupParameter": "启动参数", + "cancel": "取消", + "start": "运行", + "none_1": "都不发", + "success_1": "成功发", + "failure_1": "失败发", + "All_1": "成功或失败都发", + "receivers": "收件人", + "receiverCcs": "抄送人", + "startDate": "开始日期", + "endDate": "结束日期", + "StartAndStopTime": "起止时间", + "Timing": "定时", + "ExecuteTime": "执行时间", + "Timezone": "时区", + "fiveTimes": "接下来五次执行时间", + "create": "创建", + "edit": "编辑", + "inputEmail": "请输入邮箱,以逗号分割", + "selectTime": "请选择时间", + "startNotSameAsEnd": "开始时间和结束时间不能相同", + "enterCrontab": "请输入crontab", + "success": "成功", + "operationSuccess": "操作成功", + "PleaseGoOnline": "不要忘记上线", + "noData": "该时间段无数据" + }, + "tasksState": { + "SUBMITTED_SUCCESS": "提交成功", + "RUNNING_EXECUTION": "正在执行", + "READY_PAUSE": "准备暂停", + "PAUSE": "暂停", + "READY_STOP": "准备停止", + "STOP": "停止", + "FAILURE": "失败", + "SUCCESS": "成功", + "NEED_FAULT_TOLERANCE": "需要容错", + "KILL": "Kill", + "WAITTING_THREAD": "等待线程", + "WAITTING_DEPEND": "等待依赖", + "DELAY_EXECUTION": "延时执行", + "FORCED_SUCCESS": "强制成功" + }, + "header": { + "id": "编号", + "ProcessName": "工作流名称", + "ProcessInstance": "工作流实例", + "nodeType": "节点类型", + "Name": "名称", + "Number": "数量", + "State": "状态", + "RunType": "运行类型", + "SchedulingTime": "调度时间", + "SubmitTime": "提交时间", + "StartTime": "开始时间", + "EndTime": "结束时间", + "Duration": "运行时长", + "RunTimes": "运行次数", + "retryTimes": "重试次数", + "FaultTolerantSign": "容错标识", + "Executor": "执行用户", + "Operation": "操作", + "CreateTime": "创建时间", + "UpdateTime": "更新时间", + "Description": "描述", + "ModifyBy": "修改用户", + "TimingState": "定时状态", + "crontab": "crontab", + "failureStrategy": "失败策略" + } + } + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/common.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/common.js new file mode 100644 index 000000000..018f63849 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/common.js @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Find the container and initialize the chart according to the parameters, and then return one or a group of chart instances + * @param {*} Target Chart component class + * @param {*} el Selector or DOM object + * @param {*} data data source + * @param {*} options Optional + */ +export const init = (Target, el, data, options) => { + const list = getChartContainers(el) + const settings = Object.assign({}, { data }, options) + const charts = list.map(element => { + return new Target(element, settings) + }) + return charts.length === 1 ? charts[0] : charts +} + +/** + * Unified chart container is an array of DOM elements + * @param {*} el Selector or DOM object + */ +function getChartContainers (el) { + // Parameter not transmitted, return directly + if (!el) { + return + } + if (typeof el === 'string') { + if (el.startsWith('#')) { + el = document.getElementById(el.slice(1)) + } else if (el.startsWith('.')) { + el = document.getElementsByClassName(el.slice(1)) + } else { + return + } + } + if (!el) { + throw new Error('No corresponding DOM object found!') + } + let list + if (Object.prototype.isPrototypeOf.call(HTMLElement.prototype, el)) { + list = new Array(el) + } else { + list = Array.from(el) + } + if (!list) { + throw new Error('No corresponding DOM object found!') + } + return list +} + +/** + * Detects whether the specified property name exists in the specified object + * @param {Object} model Model to be tested + * @param {...any} params Property name to be tested + */ +export const checkKeyInModel = (model, ...params) => { + for (const key of params) { + if (!Object.prototype.hasOwnProperty.call(model, key)) { + throw new Error('Data format error! The specified property was not found:' + key) + } + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/index.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/index.js new file mode 100644 index 000000000..5f051c255 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/index.js @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import echarts from 'echarts' + +import Line from './packages/line' +import Bar from './packages/bar' +import Pie from './packages/pie' +import Radar from './packages/radar' +import Funnel from './packages/funnel' +import Scatter from './packages/scatter' +import { checkKeyInModel, init } from './common' + +const components = { + Line, + Bar, + Pie, + Radar, + Funnel, + Scatter +} + +const Chart = { + // Default configuration + settings: {}, + name: 'ana-chart', + /** + * Configure global properties + * @param {Object} options Global configuration item + */ + config (options) { + const { theme } = options + // Registration theme + if (theme) { + checkKeyInModel(theme, 'name', 'data') + echarts.registerTheme(theme.name, theme.data) + if (theme.default) { + Chart.settings.defaultTheme = theme.name + } + } + } +} + +// Corresponding methods for injection of different components +for (const key in components) { + if (Object.prototype.hasOwnProperty.call(components, key)) { + Chart[key.toLowerCase()] = (el, data, options) => { + return init(components[key], el, data, options) + } + } +} + +export { + Line, + Bar, + Pie, + Radar, + Funnel, + Scatter +} + +export default Chart + +if (typeof window !== 'undefined') { + window.Chart = Chart +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/bar/index.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/bar/index.js new file mode 100644 index 000000000..65e58ef20 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/bar/index.js @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Base from '../base' +import { checkKeyInModel, init } from '../../common' + +const TYPE = 'bar' + +/** + * Histogram + */ +export default class Bar extends Base { + /** + * Initialization method called on separate export + * @param {*} el Selector or DOM object + * @param {*} data data source + * @param {*} options Optional + */ + static init (el, data, options) { + return init(Bar, el, data, options) + } + + /** + * Convert user configuration to a configuration format that conforms to the format of echarts API + */ + transform () { + const { data = [] } = this.settings + + if (data.length === 0) { + throw new Error('Data source is empty!') + } + + if (Object.keys(data[0]).length > 2) { + return this.setMultipleBars() + } else { + this.simple = true + return this.setSingleBar() + } + } + + /** + * Single column + */ + setSingleBar () { + const { + // data + data = [], + // Attribute dictionary + keyMap = { + xAxisKey: 'key', + dataKey: 'value' + }, + barColor = '', + // Chart title + title = 'Single bar histogram' + } = this.settings + + // X axis corresponds to attribute name, data value corresponds to attribute name + const { xAxisKey, dataKey } = keyMap + checkKeyInModel(data[0], xAxisKey, dataKey) + + const series = [{ + type: TYPE, + data: [], + color: barColor + }] + const xAxis = { + type: 'category', + data: [] + } + + for (let i = 0; i < data.length; i++) { + xAxis.data.push(data[i][xAxisKey]) + series[0].data.push(data[i][dataKey]) + } + + return { title, xAxis, series } + } + + /** + * Multiple columns + */ + setMultipleBars () { + const { + // data + data = [], + // Attribute dictionary + keyMap = { + xAxisKey: 'key', + legendKey: 'typeName', + dataKey: 'value' + }, + // Chart title + title = 'Multiple histogram', + // The specified index of polyline data when the column is mixed + lineTypes + } = this.settings + + // Attribute name corresponding to X axis, legend and data value + const { xAxisKey, legendKey, dataKey } = keyMap + // Use timeline data or not + const timeline = Object.keys(data[0]).length === 4 + const timelineKey = keyMap.timelineKey || 'timeline' + if (timeline) { + checkKeyInModel(data[0], xAxisKey, legendKey, dataKey, timelineKey) + } else { + checkKeyInModel(data[0], xAxisKey, legendKey, dataKey) + } + + // Standard mixed index of folded columns + let lineTypeList = [] + if (lineTypes) { + if (!Array.isArray(lineTypes)) { + lineTypeList = [lineTypes] + } else { + lineTypeList = lineTypes + } + } + + // Timeline default configuration + const timelineOptions = { + timeline: { + axisType: 'category', + autoPlay: true, + playInterval: 1000, + data: [] + }, + options: [] + } + + // Initial value + const legendData = [] + const series = [] + const xAxis = { + type: 'category', + data: [] + } + for (let i = 0; i < data.length; i++) { + const legendItem = data[i][legendKey] + const xAxisItem = data[i][xAxisKey] + const dataItem = data[i][dataKey] + + // Legend + if (!legendData.includes(legendItem)) { + legendData.push(legendItem) + } + + // x axis + if (!xAxis.data.includes(xAxisItem)) { + xAxis.data.push(xAxisItem) + } + + // time axis + if (timeline) { + const timelineItem = data[i][timelineKey] + // Set timeline label + if (!timelineOptions.timeline.data.includes(timelineItem)) { + timelineOptions.timeline.data.push(timelineItem) + } + // Universal family configuration + if (!series.some(s => s.name === legendItem)) { + let seriesType = TYPE + if (lineTypeList.length !== 0 && lineTypeList.includes(legendItem)) { + seriesType = 'line' + } + series.push({ + name: legendItem, + type: seriesType + }) + } + // Series data + let targetOptions = timelineOptions.options.find(o => o._helpName === timelineItem) + if (!targetOptions) { + // Initialization option + targetOptions = { + _helpName: timelineItem, + title: { text: title.replace('$timeline', timelineItem) }, + series: [] + } + timelineOptions.options.push(targetOptions) + } + let targetSeries = targetOptions.series.find(d => d._helpName === legendItem) + if (!targetSeries) { + // Initialize series data + targetSeries = { + _helpName: legendItem, + data: [] + } + targetOptions.series.push(targetSeries) + } + targetSeries.data.push(dataItem) + } else { + // Non timeline data processing + let targetSeries = series.find(s => s.name === legendItem) + if (!targetSeries) { + let seriesType = TYPE + if (lineTypeList.length !== 0 && lineTypeList.includes(legendItem)) { + seriesType = 'line' + } + targetSeries = { + name: legendItem, + type: seriesType, + data: [] + } + series.push(targetSeries) + } + targetSeries.data.push(dataItem) + } + } + + if (timeline) { + return { title, xAxis, series, legendData, timelineOptions } + } + return { title, xAxis, series, legendData } + } + + /** + * Drawing charts + */ + apply () { + const { title, xAxis, series, legendData, timelineOptions } = this.options + const { + // Whether it is a horizontal drawing + reverseAxis = false, + // Custom Y axis + yAxis, + // Is it a stacking diagram + stack = false, + // Injection configuration to series + insertSeries + } = this.settings + const valueAxis = { type: 'value' } + let yAxisModel = reverseAxis ? xAxis : valueAxis + const xAxisModel = reverseAxis ? valueAxis : xAxis + // Use custom Y-axis overlay + if (yAxis) { + yAxisModel = yAxis + } + // Set up stack chart + if (stack) { + series.forEach(set => { + set.stack = '总量' + set.label = { + normal: { + show: true, + position: reverseAxis ? 'insideRight' : 'insideTop' + } + } + }) + } + + let _series = series + if (insertSeries && insertSeries.length && series.length) { + _series = this.injectDataIntoSeries(insertSeries, _series) + } + + // time axis + if (timelineOptions) { + const opts = { + baseOption: { + timeline: timelineOptions.timeline, + tooltip: { + trigger: 'axis' + }, + grid: { + top: 80, + bottom: 100, + containLabel: true + }, + legend: { + x: 'right', + data: legendData + }, + xAxis: xAxisModel, + yAxis: yAxisModel, + series: _series + }, + options: timelineOptions.options + } + + this.echart.setOption(opts, true) + this.echart.clear() + this.echart.setOption(opts, true) + } else { + // When the simple chart title is empty, the chart is vertically centered + const top = !title && this.simple ? '3%' : 60 + + const opts = { + title: { + text: title + }, + tooltip: { + trigger: 'axis' + }, + grid: { + left: '3%', + right: '4%', + bottom: '3%', + top, + containLabel: true + }, + legend: { + data: legendData + }, + xAxis: xAxisModel, + yAxis: yAxisModel, + series: _series + } + + this.echart.setOption(opts, true) + this.echart.clear() + this.echart.setOption(opts, true) + } + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/base.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/base.js new file mode 100644 index 000000000..c8e4839f4 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/base.js @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import echarts from 'echarts' +import Chart from '../index' + +export default class Base { + constructor (element, settings) { + this.settings = settings + const options = this.transform() + if (options) { + this.options = options + if (settings.theme) { + // Use a registered custom theme + this.echart = echarts.init(element, settings.theme) + } else if (Chart.settings.defaultTheme) { + // Topics using global configuration + this.echart = echarts.init(element, Chart.settings.defaultTheme) + } else { + this.echart = echarts.init(element) + } + // Response window resize event + window.addEventListener('resize', () => this.echart.resize()) + this.apply() + } + } + + /** + * refresh data + * @param {*} data Icon data + */ + setData (data) { + if (data && data.length !== 0) { + this.settings.data = data + this.options = this.transform() + this.apply() + } + } + + injectDataIntoSeries (data, series) { + data.forEach(o => { + if (o.index === 'all') { + delete o.index + series = series.map(item => { + return Object.assign({}, item, o) + }) + } else if (o.index === 'start') { + delete o.index + series[0] = Object.assign({}, series[0], o) + } else if (o.index === 'end') { + delete o.index + series[series.length - 1] = Object.assign({}, series[series.length - 1], o) + } else if (Array.isArray(o.index)) { + for (const i of o.index) { + if (series[i]) { + series[i] = Object.assign({}, series[i], o) + } + } + } + }) + return series + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/funnel/index.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/funnel/index.js new file mode 100644 index 000000000..5a5e243ad --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/funnel/index.js @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Base from '../base' +import { checkKeyInModel, init } from '../../common' + +const TYPE = 'funnel' + +/** + * Funnel plot + */ +export default class Funnel extends Base { + /** + * Initialization method called on separate export + * @param {*} el Selector or DOM pair + * @param {*} data data source + * @param {*} options Optional + */ + static init (el, data, options) { + return init(Funnel, el, data, options) + } + + /** + * Convert user configuration to a configuration format that conforms to the format of echarts API + */ + transform () { + const { + // data + data = [], + // title + title = 'Funnel plot', + // Attribute dictionary + keyMap = { + textKey: 'key', + dataKey: 'value' + } + } = this.settings + + if (data.length === 0) { + throw new Error('Data source is empty!') + } + + // Attribute name corresponding to text and attribute name corresponding to data value + const { textKey, dataKey } = keyMap + checkKeyInModel(data[0], textKey, dataKey) + + const legendData = [] + const series = [{ + type: TYPE, + left: '10%', + top: 60, + bottom: 60, + width: '80%', + min: 0, + max: 100, + minSize: '0%', + maxSize: '100%', + sort: 'descending', + gap: 2, + label: { + normal: { + show: true, + position: 'inside' + }, + emphasis: { + textStyle: { + fontSize: 20 + } + } + }, + labelLine: { + normal: { + length: 10, + lineStyle: { + width: 1, + type: 'solid' + } + } + }, + itemStyle: { + normal: { + borderColor: '#fff', + borderWidth: 1 + } + }, + data: [] + }] + + // Fill data + for (let i = 0; i < data.length; i++) { + const element = data[i] + const { [dataKey]: value, [textKey]: name, ...other } = element + const item = { + value, + name, + ...other, + _raw: element + } + series[0].data.push(item) + } + return { title, series, legendData } + } + + /** + * Drawing charts + */ + apply () { + const { title, series, legendData } = this.options + + // Injection configuration to series + const { insertSeries } = this.settings + let _series = series + if (insertSeries && insertSeries.length && series.length) { + _series = this.injectDataIntoSeries(insertSeries, _series) + } + + this.echart.setOption({ + title: { + text: title + }, + tooltip: { + trigger: 'item', + formatter: '{a}
{b} : {c}%' + }, + legend: { + data: legendData + }, + series: _series + }, true) + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/line/index.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/line/index.js new file mode 100644 index 000000000..94ebc8441 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/line/index.js @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Base from '../base' +import { checkKeyInModel, init } from '../../common' + +const TYPE = 'line' + +/** + * Broken line diagram + */ +export default class Line extends Base { + /** + * Initialization method called on separate export + * @param {*} el Selector or DOM object + * @param {*} data data source + * @param {*} options Optional + */ + static init (el, data, options) { + return init(Line, el, data, options) + } + + /** + * Convert user configuration to a configuration format that conforms to the format of echarts API + */ + transform () { + const { data = [] } = this.settings + + if (data.length === 0) { + throw new Error('Data source is empty!') + } + + if (Object.keys(data[0]).length > 2) { + return this.setMultipleLines() + } else { + this.simple = true + return this.setSingleLine() + } + } + + /** + * Single broken line + */ + setSingleLine () { + const { + // data + data = [], + // Attribute dictionary + keyMap = { + xAxisKey: 'key', + dataKey: 'value' + }, + // Chart title + title = 'Single line chart' + } = this.settings + + // X axis corresponds to attribute name, data value corresponds to attribute name + const { xAxisKey, dataKey } = keyMap + checkKeyInModel(data[0], xAxisKey, dataKey) + + const series = [{ + type: TYPE, + data: [] + }] + const xAxis = { + type: 'category', + data: [] + } + + for (let i = 0; i < data.length; i++) { + xAxis.data.push(data[i][xAxisKey]) + series[0].data.push(data[i][dataKey]) + } + + return { title, xAxis, series } + } + + /** + * Multiple broken lines + */ + setMultipleLines () { + const { + // data + data = [], + // Attribute dictionary + keyMap = { + xAxisKey: 'key', + legendKey: 'typeName', + dataKey: 'value' + }, + // Chart title + title = 'Multiple line chart' + } = this.settings + + // Attribute name corresponding to X axis, legend and data value + const { xAxisKey, legendKey, dataKey } = keyMap + checkKeyInModel(data[0], xAxisKey, legendKey, dataKey) + + const legendData = [] + const series = [] + const xAxis = { + type: 'category', + data: [] + } + + for (let i = 0; i < data.length; i++) { + const legendItem = data[i][legendKey] + const xAxisItem = data[i][xAxisKey] + const dataItem = data[i][dataKey] + + // Legend + if (!legendData.includes(legendItem)) { + legendData.push(legendItem) + } + + // X axis + if (!xAxis.data.includes(xAxisItem)) { + xAxis.data.push(xAxisItem) + } + + // series + let targetSeries = series.find(s => s.name === legendItem) + if (!targetSeries) { + targetSeries = { + name: legendItem, + type: TYPE, + data: [] + } + series.push(targetSeries) + } + targetSeries.data.push(dataItem) + } + + return { title, xAxis, series, legendData } + } + + /** + * Drawing charts + */ + apply () { + const { title, xAxis, series, legendData = [] } = this.options + const { + // Whether it is a horizontal drawing + reverseAxis = false, + // Custom Y axis + yAxis, + // Injection configuration to series + insertSeries + } = this.settings + const valueAxis = { type: 'value' } + let yAxisModel = reverseAxis ? xAxis : valueAxis + const xAxisModel = reverseAxis ? valueAxis : xAxis + // Use custom Y-axis overlay + if (yAxis) { + yAxisModel = yAxis + } + // When the simple chart title is empty, the chart is vertically centered + const top = !title && this.simple ? '3%' : 60 + + let _series = series + if (insertSeries && insertSeries.length && series.length) { + _series = this.injectDataIntoSeries(insertSeries, _series) + } + + const opts = { + title: { + text: title + }, + tooltip: { + trigger: 'axis' + }, + grid: { + left: '3%', + right: '4%', + bottom: '3%', + top, + containLabel: true + }, + legend: { + data: legendData + }, + xAxis: xAxisModel, + yAxis: yAxisModel, + series: _series + } + + this.echart.setOption(opts, true) + this.echart.clear() + this.echart.setOption(opts, true) + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/pie/index.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/pie/index.js new file mode 100644 index 000000000..5a3419f1c --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/pie/index.js @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Base from '../base' +import { checkKeyInModel, init } from '../../common' + +const TYPE = 'pie' + +/** + * Pie chart + */ +export default class Pie extends Base { + /** + * Initialization method called on separate export + * @param {*} el Selector or DOM object + * @param {*} data data source + * @param {*} options Optional + */ + static init (el, data, options) { + return init(Pie, el, data, options) + } + + /** + * Convert user configuration to a configuration format that conforms to the format of echarts API + */ + transform () { + const { + // data + data = [], + // title + title = 'Pie chart', + // Ring chart or not + ring = false, + // Attribute dictionary + keyMap = { + textKey: 'key', + dataKey: 'value' + } + } = this.settings + + if (data.length === 0) { + throw new Error('数据源为空!') + } + + // Attribute name corresponding to text and attribute name corresponding to data value + const { textKey, dataKey } = keyMap + checkKeyInModel(data[0], textKey, dataKey) + + const legendData = [] + const radius = ring ? ['65%', '70%'] : '60%' + const center = title ? ['50%', '60%'] : ['50%', '50%'] + const series = [{ + radius: radius, + center: center, + type: TYPE, + data: [] + }] + + // Fill data + for (let i = 0; i < data.length; i++) { + const element = data[i] + const { [dataKey]: value, [textKey]: name, ...other } = element + const item = { + value, + name, + ...other, + _raw: element + } + series[0].data.push(item) + } + return { title, series, legendData } + } + + /** + * Drawing charts + */ + apply () { + const { title, series, legendData } = this.options + + // Injection configuration to series + const { insertSeries } = this.settings + series[0].itemStyle = { + normal: { + color: (params) => { + return params.data.color + } + } + } + let _series = series + if (insertSeries && insertSeries.length && series.length) { + _series = this.injectDataIntoSeries(insertSeries, _series) + } + + const opts = { + title: { + text: title, + x: 'center', + y: 'center', + textStyle: { + fontSize: 14, + color: 'rgba(0,0,0,0.45)' + } + }, + tooltip: { + trigger: 'item', + formatter: '{b} : {c} ({d}%)' + }, + legend: { + orient: 'vertical', + left: 'left', + data: legendData + }, + series: _series + } + + this.echart.setOption(opts, true) + this.echart.clear() + this.echart.setOption(opts, true) + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/radar/index.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/radar/index.js new file mode 100644 index 000000000..375dd1caf --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/radar/index.js @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Base from '../base' +import { checkKeyInModel, init } from '../../common' + +const TYPE = 'radar' + +/** + * Radar chart + */ +export default class Radar extends Base { + /** + * Initialization method called on separate export + * @param {*} el Selector or DOM object + * @param {*} data data source + * @param {*} options Optional + */ + static init (el, data, options) { + return init(Radar, el, data, options) + } + + /** + * Convert user configuration to a configuration format that conforms to the format of echarts API + */ + transform () { + const { + // data + data = [], + // Chart title + title = 'Radar chart', + // Attribute dictionary + keyMap = { + textKey: 'key', + legendKey: 'typeName', + dataKey: 'value' + } + } = this.settings + + if (data.length === 0) { + throw new Error('Data source is empty!') + } + + // Attribute name corresponding to text, attribute name corresponding to legend, attribute name corresponding to data value + const { textKey, legendKey, dataKey } = keyMap + checkKeyInModel(data[0], textKey, legendKey, dataKey) + + const legendData = [] + const seriesData = [] + const indicator = [] + + // Set legend and initialize data series + for (let i = 0; i < data.length; i++) { + const legendItem = data[i][legendKey] + const textItem = data[i][textKey] + const dataItem = data[i][dataKey] + + // Legend + if (!legendData.includes(legendItem)) { + legendData.push(legendItem) + } + + // series + let targetSeries = seriesData.find(s => s.name === legendItem) + if (!targetSeries) { + targetSeries = { + name: legendItem, + value: [], + _raw: [] + } + seriesData.push(targetSeries) + } + targetSeries.value.push(dataItem) + targetSeries._raw.push(data[i]) + + // index + const targetIndicator = indicator.find(i => i.name === textItem) + if (!targetIndicator) { + indicator.push({ name: textItem }) + } + } + + return { title, seriesData, legendData, indicator } + } + + /** + * Drawing charts + */ + apply () { + const { title, seriesData, legendData = [], indicator } = this.options + this.echart.setOption({ + title: { + text: title + }, + tooltip: {}, + legend: { + data: legendData + }, + radar: { + name: { + textStyle: { + color: '#fff', + backgroundColor: '#999', + borderRadius: 3, + padding: [3, 5] + } + }, + indicator + }, + series: [{ + type: TYPE, + data: seriesData + }] + }, true) + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/scatter/index.js b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/scatter/index.js new file mode 100644 index 000000000..d6c09ee47 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/ana-charts/packages/scatter/index.js @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Base from '../base' +import { checkKeyInModel, init } from '../../common' + +const TYPE = 'scatter' + +/** + * Bubble chart + */ +export default class Scatter extends Base { + /** + * Initialization method called on separate export + * @param {*} el Selector or DOM object + * @param {*} data data source + * @param {*} options Optional + */ + static init (el, data, options) { + return init(Scatter, el, data, options) + } + + /** + * Convert user configuration to a configuration format that conforms to the format of echarts API + */ + transform () { + const { + // data + data = [], + // Chart title + title = 'Bubble chart', + // Attribute dictionary + keyMap = { + xKey: 'x', + yKey: 'y', + sizeKey: 'size', + textKey: 'text', + legendKey: 'typeName' + } + } = this.settings + + if (data.length === 0) { + throw new Error('Data source is empty!') + } + + const legendData = [] + const series = [] + + const { xKey, yKey, sizeKey, textKey, legendKey } = keyMap + checkKeyInModel(data[0], xKey, yKey, sizeKey, textKey, legendKey) + + for (let i = 0; i < data.length; i++) { + const { + [legendKey]: legendItem, + [xKey]: xValue, + [yKey]: yValue, + [sizeKey]: sizeValue, + [textKey]: textValue, + ...other + } = data[i] + + // Legend + if (!legendData.includes(legendItem)) { + legendData.push(legendItem) + } + + // series + let targetSeries = series.find(s => s.name === legendItem) + if (!targetSeries) { + targetSeries = { + type: TYPE, + name: legendItem, + data: [], + symbolSize: function (data) { + return Math.sqrt(data[2]) + }, + label: { + emphasis: { + show: true, + formatter: function (param) { + return param.data[3] + }, + position: 'top' + } + } + } + series.push(targetSeries) + } + targetSeries.data.push({ + value: [ + xValue, + yValue, + sizeValue, + textValue + ], + ...other, + _raw: data[i] + }) + } + + return { title, series, legendData } + } + + /** + * Drawing charts + */ + apply () { + const { title, series, legendData = [] } = this.options + + const { + // Custom X axis + xAxis, + // Custom Y axis + yAxis, + // Injection configuration to series + insertSeries + } = this.settings + let _series = series + if (insertSeries && insertSeries.length && series.length) { + _series = this.injectDataIntoSeries(insertSeries, _series) + } + + this.echart.setOption({ + title: { + text: title + }, + legend: { + right: 10, + data: legendData + }, + xAxis: xAxis || {}, + yAxis: yAxis || {}, + series: _series + }, true) + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/components/listConstruction/listConstruction.vue b/web/packages/dolphinScheduler/module/dispatch/components/listConstruction/listConstruction.vue new file mode 100644 index 000000000..8c8053caf --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/listConstruction/listConstruction.vue @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/components/noData/images/errorTip.png b/web/packages/dolphinScheduler/module/dispatch/components/noData/images/errorTip.png new file mode 100644 index 000000000..775df073a Binary files /dev/null and b/web/packages/dolphinScheduler/module/dispatch/components/noData/images/errorTip.png differ diff --git a/web/packages/dolphinScheduler/module/dispatch/components/noData/images/void_page.png b/web/packages/dolphinScheduler/module/dispatch/components/noData/images/void_page.png new file mode 100644 index 000000000..633bb6cf0 Binary files /dev/null and b/web/packages/dolphinScheduler/module/dispatch/components/noData/images/void_page.png differ diff --git a/web/packages/dolphinScheduler/module/dispatch/components/noData/noData.vue b/web/packages/dolphinScheduler/module/dispatch/components/noData/noData.vue new file mode 100644 index 000000000..f0015c278 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/noData/noData.vue @@ -0,0 +1,44 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/components/spin/spin.vue b/web/packages/dolphinScheduler/module/dispatch/components/spin/spin.vue new file mode 100644 index 000000000..7695cf051 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/components/spin/spin.vue @@ -0,0 +1,64 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/config.js b/web/packages/dolphinScheduler/module/dispatch/config.js new file mode 100644 index 000000000..b6fae3db6 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/config.js @@ -0,0 +1,354 @@ +import i18n from '@dataspherestudio/shared/common/i18n' +i18n.$t = i18n.t +const tasksState = { + SUBMITTED_SUCCESS: { + id: 0, + desc: i18n.$t('message.scheduler.tasksState.SUBMITTED_SUCCESS'), + icon: 'icon-submitted-success', + color: '#A9A9A9' + }, + 'RUNNING_EXECUTION': { + id: 1, + desc: i18n.$t('message.scheduler.tasksState.RUNNING_EXECUTION'), + icon: 'icon-running-execution', + color: '#0097e0' + }, + 'RUNNING_EXEUTION': { + id: 100, + desc: i18n.$t('message.scheduler.tasksState.RUNNING_EXECUTION'), + icon: 'icon-running-execution', + color: '#0097e0' + }, + READY_PAUSE: { + id: 2, + desc: i18n.$t('message.scheduler.tasksState.READY_PAUSE'), + icon: 'icon-ready-pause', + color: '#07b1a3' + }, + PAUSE: { + id: 3, + desc: i18n.$t('message.scheduler.tasksState.PAUSE'), + icon: 'icon-pause', + color: '#057c72' + }, + READY_STOP: { + id: 4, + desc: i18n.$t('message.scheduler.tasksState.READY_STOP'), + icon: 'icon-ready-stop', + color: '#FE0402' + }, + STOP: { + id: 5, + desc: i18n.$t('message.scheduler.tasksState.STOP'), + icon: 'icon-stop', + color: '#e90101' + }, + FAILURE: { + id: 6, + desc: i18n.$t('message.scheduler.tasksState.FAILURE'), + icon: 'icon-failure', + color: '#000000' + }, + SUCCESS: { + id: 7, + desc: i18n.$t('message.scheduler.tasksState.SUCCESS'), + icon: 'icon-success', + color: '#33cc00' + }, + NEED_FAULT_TOLERANCE: { + id: 8, + desc: i18n.$t('message.scheduler.tasksState.NEED_FAULT_TOLERANCE'), + icon: 'icon-need-fault-tolerance', + color: '#FF8C00' + }, + KILL: { + id: 9, + desc: i18n.$t('message.scheduler.tasksState.KILL'), + icon: 'icon-kill', + color: '#a70202' + }, + WAITTING_THREAD: { + id: 10, + desc: i18n.$t('message.scheduler.tasksState.WAITTING_THREAD'), + icon: 'icon-waitting-thread', + color: '#912eed' + }, + WAITTING_DEPEND: { + id: 11, + desc: i18n.$t('message.scheduler.tasksState.WAITTING_DEPEND'), + icon: 'icon-watting-depend', + color: '#5101be' + }, + DELAY_EXECUTION: { + id: 12, + desc: i18n.$t('message.scheduler.tasksState.DELAY_EXECUTION'), + icon: 'icon-delay-execution', + color: '#5102ce' + }, + FORCED_SUCCESS: { + id: 13, + desc: i18n.$t('message.scheduler.tasksState.FORCED_SUCCESS'), + icon: 'icon-forced-success', + color: '#5102ce' + } +} + +const tasksStateList = [ + { + id: 0, + desc: i18n.$t('message.scheduler.tasksState.SUBMITTED_SUCCESS'), + icon: 'icon-submitted-success', + code: 'SUBMITTED_SUCCESS', + color: '#A9A9A9' + }, + { + id: 1, + desc: i18n.$t('message.scheduler.tasksState.RUNNING_EXECUTION'), + icon: 'icon-running-execution', + code: 'RUNNING_EXEUTION', + color: '#0097e0' + }, + { + id: 2, + desc: i18n.$t('message.scheduler.tasksState.READY_PAUSE'), + icon: 'icon-ready-pause', + code: 'READY_PAUSE', + color: '#07b1a3' + }, + { + id: 3, + desc: i18n.$t('message.scheduler.tasksState.PAUSE'), + icon: 'icon-pause', + code: 'PAUSE', + color: '#057c72' + }, + { + id: 4, + desc: i18n.$t('message.scheduler.tasksState.READY_STOP'), + icon: 'icon-ready-stop', + code: 'READY_STOP', + color: '#FE0402' + }, + { + id: 5, + desc: i18n.$t('message.scheduler.tasksState.STOP'), + icon: 'icon-stop', + code: 'STOP', + color: '#e90101' + }, + { + id: 6, + desc: i18n.$t('message.scheduler.tasksState.FAILURE'), + icon: 'icon-failure', + code: 'FAILURE', + color: '#000000' + }, + { + id: 7, + desc: i18n.$t('message.scheduler.tasksState.SUCCESS'), + icon: 'icon-success', + code: 'SUCCESS', + color: '#33cc00' + }, + { + id: 8, + desc: i18n.$t('message.scheduler.tasksState.NEED_FAULT_TOLERANCE'), + icon: 'icon-need-fault-tolerance', + code: 'NEED_FAULT_TOLERANCE', + color: '#FF8C00' + }, + { + id: 9, + desc: i18n.$t('message.scheduler.tasksState.KILL'), + icon: 'icon-kill', + code: 'KILL', + color: '#a70202' + }, + { + id: 10, + desc: i18n.$t('message.scheduler.tasksState.WAITTING_THREAD'), + icon: 'icon-waitting-thread', + code: 'WAITTING_THREAD', + color: '#912eed' + }, + { + id: 11, + desc: i18n.$t('message.scheduler.tasksState.WAITTING_DEPEND'), + icon: 'icon-watting-depend', + code: 'WAITTING_DEPEND', + color: '#5101be' + } +] + +const dashboardStateList = [ + { + id: 1, + desc: i18n.$t('message.scheduler.tasksState.RUNNING_EXECUTION'), + icon: 'icon-running-execution', + code: 'RUNNING_EXEUTION', + color: '#73A0FA' + }, + { + id: 3, + desc: i18n.$t('message.scheduler.tasksState.PAUSE'), + icon: 'icon-pause', + code: 'PAUSE', + color: '#F9D66F' + }, + { + id: 5, + desc: i18n.$t('message.scheduler.tasksState.STOP'), + icon: 'icon-stop', + code: 'STOP', + color: '#EB7E65' + }, + { + id: 6, + desc: i18n.$t('message.scheduler.tasksState.FAILURE'), + icon: 'icon-failure', + code: 'FAILURE', + color: '#838383' + }, + { + id: 7, + desc: i18n.$t('message.scheduler.tasksState.SUCCESS'), + icon: 'icon-success', + code: 'SUCCESS', + color: '#73DEB3' + } +] + +const publishStatus = { + 'NOT_RELEASE': i18n.$t('message.scheduler.Unpublished'), + 'ONLINE': i18n.$t('message.scheduler.online'), + 'OFFLINE': i18n.$t('message.scheduler.offline') +} + +const runningType = [ + { + desc: i18n.$t('message.scheduler.START_PROCESS'), + code: 'START_PROCESS' + }, + { + desc: i18n.$t('message.scheduler.START_CURRENT_TASK_PROCESS'), + code: 'START_CURRENT_TASK_PROCESS' + }, + { + desc: i18n.$t('message.scheduler.RECOVER_TOLERANCE_FAULT_PROCESS'), + code: 'RECOVER_TOLERANCE_FAULT_PROCESS' + }, + { + desc: i18n.$t('message.scheduler.RECOVER_SUSPENDED_PROCESS'), + code: 'RECOVER_SUSPENDED_PROCESS' + }, + { + desc: i18n.$t('message.scheduler.START_FAILURE_TASK_PROCESS'), + code: 'START_FAILURE_TASK_PROCESS' + }, + { + desc: i18n.$t('message.scheduler.COMPLEMENT_DATA'), + code: 'COMPLEMENT_DATA' + }, + { + desc: i18n.$t('message.scheduler.SCHEDULER'), + code: 'SCHEDULER' + }, + { + desc: i18n.$t('message.scheduler.rerun'), + code: 'REPEAT_RUNNING' + }, + { + desc: i18n.$t('message.scheduler.PAUSE'), + code: 'PAUSE' + }, + { + desc: i18n.$t('message.scheduler.STOP'), + code: 'STOP' + }, + { + desc: i18n.$t('message.scheduler.RECOVER_WAITTING_THREAD'), + code: 'RECOVER_WAITTING_THREAD' + } +] + +const tasksType = { + SHELL: { + desc: 'SHELL', + color: '#646464', + icon: 'icon-shell' + }, + WATERDROP: { + desc: 'WATERDROP', + color: '#646465', + icon: 'icon-shell' + }, + SUB_PROCESS: { + desc: 'SUB_PROCESS', + color: '#0097e0', + icon: 'icon-shell' + }, + PROCEDURE: { + desc: 'PROCEDURE', + color: '#525CCD', + icon: 'icon-shell' + }, + SQL: { + desc: 'SQL', + color: '#7A98A1', + icon: 'icon-shell' + }, + SPARK: { + desc: 'SPARK', + color: '#E46F13', + icon: 'icon-shell' + }, + FLINK: { + desc: 'FLINK', + color: '#E46F13', + icon: 'icon-shell' + }, + MR: { + desc: 'MapReduce', + color: '#A0A5CC', + icon: 'icon-shell' + }, + PYTHON: { + desc: 'PYTHON', + color: '#FED52D', + icon: 'icon-shell' + }, + DEPENDENT: { + desc: 'DEPENDENT', + color: '#2FBFD8', + icon: 'icon-shell' + }, + HTTP: { + desc: 'HTTP', + color: '#E46F13', + icon: 'icon-shell' + }, + DATAX: { + desc: 'DataX', + color: '#1fc747', + icon: 'icon-shell' + }, + SQOOP: { + desc: 'SQOOP', + color: '#E46F13', + icon: 'icon-shell' + }, + CONDITIONS: { + desc: 'CONDITIONS', + color: '#E46F13', + icon: 'icon-shell' + } +} + +export { + tasksState, + tasksStateList, + publishStatus, + runningType, + tasksType, + dashboardStateList +} diff --git a/web/packages/dolphinScheduler/module/dispatch/convertor.js b/web/packages/dolphinScheduler/module/dispatch/convertor.js new file mode 100644 index 000000000..589293e80 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/convertor.js @@ -0,0 +1,142 @@ +import dayjs from 'dayjs' + +export function ds2butterfly(tasks, connects, locations, taskList, isNode, contextMenu) { + let nodes = [], + edges = [] + tasks.forEach(task => { + nodes.push({ + id: task.id, + left: locations[task.id].x, + top: locations[task.id].y, + label: task.name, + node_type: task.type, + node_params: task.params, + node_preTasks: task.preTasks, + contextMenu: contextMenu, + state: '', + endpoints: isNode ? [] : [ + { + id: 'top-' + task.id, + orientation: [0, -1], + pos: [0.5, 0] + }, + { + id: 'bottom-' + task.id, + orientation: [0, 1], + pos: [0.5, 0] + }, + { + id: 'left-' + task.id, + orientation: [-1, 0], + pos: [0, 0.5] + }, + { + id: 'right-' + task.id, + orientation: [1, 0], + pos: [0, 0.5] + } + ] + }) + }) + connects.forEach(connect => { + let preId = connect.endPointSourceId, + curId = connect.endPointTargetId, + curNode, preNode, prePoint, curPoint + if (!isNode) { + nodes.forEach(node => { + if (node.id === preId) { + preNode = node + } else if (node.id === curId) { + curNode = node + } + }) + if (!preNode || !curNode) return + if (preNode.left < curNode.left) { + if (preNode.top < curNode.top) { + if (Math.abs(curNode.top - preNode.top) < Math.abs(curNode.left - preNode.left)) { + prePoint = 'right-' + preId + curPoint = 'left-' + curId + } else { + prePoint = 'bottom-' + preId + curPoint = 'top-' + curId + } + } else { + if (Math.abs(curNode.top - preNode.top) < Math.abs(curNode.left - preNode.left)) { + prePoint = 'right-' + preId + curPoint = 'left-' + curId + } else { + prePoint = 'top-' + preId + curPoint = 'bottom-' + curId + } + } + } else { + if (preNode.top < curNode.top) { + if (Math.abs(curNode.top - preNode.top) < Math.abs(curNode.left - preNode.left)) { + prePoint = 'left-' + preId + curPoint = 'right-' + curId + } else { + prePoint = 'bottom-' + preId + curPoint = 'top-' + curId + } + } else { + if (Math.abs(curNode.top - preNode.top) < Math.abs(curNode.left - preNode.left)) { + prePoint = 'left-' + preId + curPoint = 'right-' + curId + } else { + prePoint = 'top-' + preId + curPoint = 'bottom-' + curId + } + } + } + } + edges.push({ + type: isNode ? 'node' : 'endpoint',//默认 + shapeType: 'AdvancedBezier', //默认 + sourceNode: isNode ? '' : preId, //连接源节点id + source: isNode ? preId : prePoint, //连接源锚点id + targetNode: isNode ? '' : curId, //连接目标节点id + target: isNode ? curId : curPoint, //连接目标锚点id + arrow: true, + arrowPosition: 1, + arrowOffset: 0, + orientationLimit: ['Left', 'Right', 'Top', 'Bottom'] + }) + }) + if (taskList && taskList.length) { + taskList.forEach(item => { + nodes.forEach(node => { + if (node.id === item.taskObj.id && item.stateObj) { + node.state = item.stateObj + } + }) + }) + } + return { + nodes, + edges + } +} + +function formatISODate (date) { + let [datetime, timezone] = date.split('+') + if (!timezone || timezone.indexOf(':') >= 0) return date + let hourOfTz = timezone.substring(0, 2) || '00' + let secondOfTz = timezone.substring(2, 4) || '00' + return `${datetime}+${hourOfTz}:${secondOfTz}` +} +export function formatDate(value, fmt) { + fmt = fmt || 'YYYY-MM-DD HH:mm:ss' + if (value === null) { + return '-' + } else { + return dayjs(formatISODate(value)).format(fmt) + } +} + +export function filterNull(value) { + if (value === null || value === '') { + return '-' + } else { + return value + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/index.js b/web/packages/dolphinScheduler/module/dispatch/crontab/index.js new file mode 100644 index 000000000..c9d96fd59 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/index.js @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import vCrontab from './source/app.vue' +// import './source/index.scss' + +export default vCrontab diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/config.js b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/config.js new file mode 100644 index 000000000..af192d86d --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/config.js @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* eslint-disable */ +import _ from 'lodash' +import zh_CN from './locale/zh_CN' +import en_US from './locale/en_US' + +export function localeList () { + return [ + { + code: 'zh_CN', + name: '中文', + locale: zh_CN + }, + { + code: 'en_US', + name: 'English', + locale: en_US + }, + { + code: 'zh-CN', + name: '中文', + locale: zh_CN + }, + { + code: 'en', + name: 'English', + locale: en_US + } + ] +} + +export function findLocale (code) { + return _.find(localeList(), ['code', code]) +} diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/index.js b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/index.js new file mode 100644 index 000000000..db7a4151a --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/index.js @@ -0,0 +1,28 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { findLocale } from './config' +import { template } from './util' + +export default { + name: '$t2', + methods: { + $t2 (str, data) { + return template(findLocale(window.localeCrontab).locale[str], data) + } + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/locale/en_US.js b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/locale/en_US.js new file mode 100644 index 000000000..ce067ffcf --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/locale/en_US.js @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export default { + 秒: 'second', + 分: 'minute', + 时: 'hour', + 天: 'day', + 月: 'month', + 年: 'year', + 星期一: 'Monday', + 星期二: 'Tuesday', + 星期三: 'Wednesday', + 星期四: 'Thursday', + 星期五: 'Friday', + 星期六: 'Saturday', + 星期天: 'Sunday', + 每一秒钟: 'Every second', + 每隔: 'Every', + '秒执行 从': 'second carried out', + 秒开始: 'Start', + '具体秒数(可多选)': 'Specific second(multiple)', + 请选择具体秒数: 'Please enter a specific second', + 周期从: 'Cycle from', + 到: 'to', + 每一分钟: 'Every minute', + '分执行 从': 'minute carried out', + 分开始: 'Start', + '具体分钟数(可多选)': 'Specific minute(multiple)', + 请选择具体分钟数: 'Please enter a specific minute', + 每一小时: 'Every hour', + '小时执行 从': 'hour carried out', + 小时开始: 'Start', + '具体小时数(可多选)': 'Specific hour(multiple)', + 请选择具体小时数: 'Please enter a hour', + 每一天: 'Every day', + '周执行 从': 'week carried out', + 开始: 'Start', + '天执行 从': 'day carried out', + 天开始: 'Start', + '具体星期几(可多选)': 'Specific day of the week(multiple)', + 请选择具体周几: 'Please enter a week', + '具体天数(可多选)': 'Specific days(multiple)', + 请选择具体天数: 'Please enter a days', + 在这个月的最后一天: 'On the last day of the month', + 在这个月的最后一个工作日: 'On the last working day of the month', + 在这个月的最后一个: 'At the last of this month', + 在本月底前: 'Before the end of this month', + '最近的工作日(周一至周五)至本月': 'The most recent business day (Monday to Friday) to this month', + 在这个月的第: 'In this months', + 每一月: 'Every month', + '月执行 从': 'month carried out', + 月开始: 'Start', + '具体月数(可多选)': 'Specific months(multiple)', + 请选择具体月数: 'Please enter a months', + 每一年: 'Every year', + '年执行 从': 'year carried out', + 年开始: 'Start', + '具体年数(可多选)': 'Specific year(multiple)', + 请选择具体年数: 'Please enter a year', + 小时: 'hour', + 日: 'day' +} diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/locale/zh_CN.js b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/locale/zh_CN.js new file mode 100644 index 000000000..c84939fc5 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/locale/zh_CN.js @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export default { + 秒: '秒', + 分: '分', + 时: '时', + 天: '天', + 月: '月', + 年: '年', + 星期一: '星期一', + 星期二: '星期二', + 星期三: '星期三', + 星期四: '星期四', + 星期五: '星期五', + 星期六: '星期六', + 星期天: '星期天', + 每一秒钟: '每一秒钟', + 每隔: '每隔', + '秒执行 从': '秒执行 从', + 秒开始: '秒开始', + '具体秒数(可多选)': '具体秒数(可多选)', + 请选择具体秒数: '请选择具体秒数', + 周期从: '周期从', + 到: '到', + 每一分钟: '每一分钟', + '分执行 从': '分执行 从', + 分开始: '分开始', + '具体分钟数(可多选)': '具体分钟数(可多选)', + 请选择具体分钟数: '请选择具体分钟数', + 每一小时: '每一小时', + '小时执行 从': '小时执行 从', + 小时开始: '小时开始', + '具体小时数(可多选)': '具体小时数(可多选)', + 请选择具体小时数: '请选择具体小时数', + 每一天: '每一天', + '周执行 从': '周执行 从', + 开始: '开始', + '天执行 从': '天执行 从', + 天开始: '天开始', + '具体星期几(可多选)': '具体星期几(可多选)', + 请选择具体周几: '请选择具体周几', + '具体天数(可多选)': '具体天数(可多选)', + 请选择具体天数: '请选择具体天数', + 在这个月的最后一天: '在这个月的最后一天', + 在这个月的最后一个工作日: '在这个月的最后一个工作日', + 在这个月的最后一个: '在这个月的最后一个', + 在本月底前: '在本月底前', + '最近的工作日(周一至周五)至本月': '最近的工作日(周一至周五)至本月', + 在这个月的第: '在这个月的第', + 每一月: '每一月', + '月执行 从': '月执行 从', + 月开始: '月开始', + '具体月数(可多选)': '具体月数(可多选)', + 请选择具体月数: '请选择具体月数', + 每一年: '每一年', + '年执行 从': '年执行 从', + 年开始: '年开始', + '具体年数(可多选)': '具体年数(可多选)', + 请选择具体年数: '请选择具体年数', + 小时: '小时', + 日: '日' +} diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/util.js b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/util.js new file mode 100644 index 000000000..7fb17b3b1 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/i18n/util.js @@ -0,0 +1,50 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * template + * + * @param {String} string + * @param {Array} ...args + * @return {String} + */ +const { hasOwnProperty } = {} +const RE_NARGS = /(%|)\{([0-9a-zA-Z_]+)\}/g +const hasOwn = (o, k) => hasOwnProperty.call(o, k) +const template = (string, ...args) => { + if (args.length === 1 && typeof args[0] === 'object') { + args = args[0] + } + if (!args || !args.hasOwnProperty) { + args = {} + } + return string.replace(RE_NARGS, (match, prefix, i, index) => { + let result + if (string[index - 1] === '{' && + string[index + match.length] === '}') { + return i + } else { + result = hasOwn(args, i) ? args[i] : null + if (result === null || result === undefined) { + return '' + } + return result + } + }) +} + +export { template } diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/input-number.vue b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/input-number.vue new file mode 100644 index 000000000..7add06f1c --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_source/input-number.vue @@ -0,0 +1,105 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/day.vue b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/day.vue new file mode 100644 index 000000000..6b09fd344 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/day.vue @@ -0,0 +1,513 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/hour.vue b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/hour.vue new file mode 100644 index 000000000..5a97bc42c --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/hour.vue @@ -0,0 +1,251 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/minute.vue b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/minute.vue new file mode 100644 index 000000000..ebc49959c --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/minute.vue @@ -0,0 +1,252 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/month.vue b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/month.vue new file mode 100644 index 000000000..f23b3ee56 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/month.vue @@ -0,0 +1,251 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/second.vue b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/second.vue new file mode 100644 index 000000000..b2ba52b94 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/second.vue @@ -0,0 +1,256 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/year.vue b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/year.vue new file mode 100644 index 000000000..9e3b678b2 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/_times/year.vue @@ -0,0 +1,249 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/app.vue b/web/packages/dolphinScheduler/module/dispatch/crontab/source/app.vue new file mode 100644 index 000000000..f7f9a1ab2 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/app.vue @@ -0,0 +1,184 @@ + + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/index.scss b/web/packages/dolphinScheduler/module/dispatch/crontab/source/index.scss new file mode 100644 index 000000000..0d60b8a7d --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/index.scss @@ -0,0 +1,61 @@ +.v-crontab-form-model { + width: 525px; + .list-box { + //padding: 6px 0; + height: 40px; + .ans-radio-wrapper { + height: auto !important; + } + .ans-radio-group-item { + .text { + color: #888; + } + } + .ans-radio-wrapper-checked { + .text { + color: #0097e0; + } + } + .ans-select { + .tag-container { + .tag-wrapper { + line-height: 10px; + margin-left: 6px; + .tag-text { + margin-right: 0; + } + } + } + } + } + .ivu-select{ + width: 300px; + } + .input-number-model{ + margin-left: 5px; + width: 100px; + button { + padding: 1px 5px; + color: #515a6e; + background-color: #f7f7f7; + border-color: rgb(220, 222, 226); + } + :first-child { + border-top-right-radius: 0; + border-bottom-right-radius: 0; + span { + padding: 0 1px; + } + } + :nth-child(3) { + border-top-left-radius: 0; + border-bottom-left-radius: 0; + } + input { + border-radius: 0; + height: 23px; + border-right: none; + border-left: none; + } + } +} diff --git a/web/packages/dolphinScheduler/module/dispatch/crontab/source/util/index.js b/web/packages/dolphinScheduler/module/dispatch/crontab/source/util/index.js new file mode 100644 index 000000000..c67b7d2e2 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/crontab/source/util/index.js @@ -0,0 +1,218 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*eslint-disable */ +import _ from 'lodash' + +/** + * Generate specified interval number + * @param start Starting value + * @param end End value + */ +const range = (start, end) => { + const length = end - start + 1 + let step = start - 1 + return Array.apply(null, { length: length }).map(function (v, i) { step++; return step }) +} + +const selectList = { + 60: _.map(range(0, 59), v => { + return { + value: v + '', + label: v + '' + } + }), + 24: _.map(range(0, 23), v => { + return { + value: v + '', + label: v + '' + } + }), + 12: _.map(range(0, 12), v => { + return { + value: v + '', + label: v + '' + } + }), + year: _.map(range(2018, 2030), v => { + return { + value: v + '', + label: v + '' + } + }), + week: [ + { + value: 1, + label: '星期天' + }, + { + value: 2, + label: '星期一' + }, + { + value: 3, + label: '星期二' + }, + { + value: 4, + label: '星期三' + }, + { + value: 5, + label: '星期四' + }, + { + value: 6, + label: '星期五' + }, + { + value: 7, + label: '星期六' + } + ], + specificWeek: [ + { + value: 'SUN', + label: 'SUN' + }, + { + value: 'MON', + label: 'MON' + }, + { + value: 'TUE', + label: 'TUE' + }, + { + value: 'WED', + label: 'WED' + }, + { + value: 'THU', + label: 'THU' + }, + { + value: 'FRI', + label: 'FRI' + }, + { + value: 'SAT', + label: 'SAT' + } + ], + day: _.map(range(1, 31), v => { + return { + value: v + '', + label: v + '' + } + }), + lastWeeks: [ + { + value: '?', + label: '星期天' + }, + { + value: '2L', + label: '星期一' + }, + { + value: '3L', + label: '星期二' + }, + { + value: '4L', + label: '星期三' + }, + { + value: '5L', + label: '星期四' + }, + { + value: '6L', + label: '星期五' + }, + { + value: '7L', + label: '星期六' + } + ] +} + +const isStr = (str, v) => { + let flag + if (str.indexOf(v) !== -1) { + flag = str.split(v) + } + return flag +} + +const isWeek = (str) => { + let flag = false + const data = str.split(',') + const isSpecificWeek = (key) => { + return _.findIndex(selectList.specificWeek, v => v.value === key) !== -1 + } + _.map(data, v => { + if (isSpecificWeek(v)) { + flag = true + } + }) + return flag +} + +/** + * template + * + * @param {String} string + * @param {Array} ...args + * @return {String} + */ +const { hasOwnProperty } = {} +const RE_NARGS = /(%|)\{([0-9a-zA-Z_]+)\}/g +const hasOwn = (o, k) => hasOwnProperty.call(o, k) +const template = (string, ...args) => { + if (args.length === 1 && typeof args[0] === 'object') { + args = args[0] + } + if (!args || !args.hasOwnProperty) { + args = {} + } + return string.replace(RE_NARGS, (match, prefix, i, index) => { + let result + if (string[index - 1] === '{' && + string[index + match.length] === '}') { + return i + } else { + result = hasOwn(args, i) ? args[i] : null + if (result === null || result === undefined) { + return '' + } + return result + } + }) +} + +const name = 'crontab-util' + +export { + name, + selectList, + isStr, + isWeek, + template +} diff --git a/web/packages/dolphinScheduler/module/dispatch/dag/index.js b/web/packages/dolphinScheduler/module/dispatch/dag/index.js new file mode 100644 index 000000000..cf5318144 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dag/index.js @@ -0,0 +1,2 @@ +import dag from './index.vue'; +export default dag; diff --git a/web/packages/dolphinScheduler/module/dispatch/dag/index.vue b/web/packages/dolphinScheduler/module/dispatch/dag/index.vue new file mode 100644 index 000000000..9d608f654 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dag/index.vue @@ -0,0 +1,82 @@ + + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/dag/source/normal-node.js b/web/packages/dolphinScheduler/module/dispatch/dag/source/normal-node.js new file mode 100644 index 000000000..12bc5625e --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dag/source/normal-node.js @@ -0,0 +1,67 @@ +/*eslint-disable */ + +import {Node} from 'butterfly-dag' +import {tasksType} from '../../config' + +import './normal-node.scss' +import util from '@dataspherestudio/shared/common/util' + +class NormalNode extends Node { + constructor(opts) { + super(opts); + this.options = opts; + } + + draw = (opts) => { + const $contextmenu = $('#rightMenu') + let container = $('
') + .css('top', opts.top) + .css('left', opts.left) + .attr('id', opts.id) + .addClass(opts.options.className) + .on('contextmenu', (e) => { + const html = [ + `` + ] + const operationHtml = () => { + return html.splice(',') + } + $contextmenu.css({ + left: e.pageX + 5, + top: e.pageY + 5, + visibility: 'visible', + textAlign: 'center' + }) + // Action bar + $contextmenu.html('').append(operationHtml) + $('#rmViewLog').on('click', () => { + util.Hub.$emit('dagLog', opts.options) + }) + return false + }) + let icon = '', + state = '' + if (opts.options.node_type) { + let color = tasksType[opts.options.node_type].color, + iconfont = tasksType[opts.options.node_type].icon + icon = `` + } + if (opts.options.state) { + state = `` + } + let logoContainer = $(`
${icon}${opts.options.label}${state}
`); + logoContainer.addClass(opts.options.className); + + container.append(logoContainer) + + let body = $('body').on('click', () => { + $contextmenu.css({ + visibility: 'hidden' + }) + }) + + return container[0]; + } +} + +export default NormalNode diff --git a/web/packages/dolphinScheduler/module/dispatch/dag/source/normal-node.scss b/web/packages/dolphinScheduler/module/dispatch/dag/source/normal-node.scss new file mode 100644 index 000000000..4b4a5dede --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dag/source/normal-node.scss @@ -0,0 +1,43 @@ +.dag-page { + .relation-node { + position: absolute; + width: 140px; + height: 36px; + padding: 0; + box-shadow:0 2px 3px 0 rgba(0,112,204,0.06); + border-radius:100px; + border:1px solid #D9D9D9; + cursor: pointer; + &:hover { + border:1px solid #F66902; + .datac-icon { + line-height: 52px; + } + } + .logo-container { + width: 100%; + height: 100%; + border: none; + line-height: 36px; + text-align: center; + margin: auto; + display: inherit; + background: none; + } + } + + .datac-icon { + font-size: 35px; + line-height: 60px; + } + .long-text { + text-align: center; + } + .nodeBackground-color{ + background-color:rgba(255, 255, 255, 0.8); + } +} + +.menu-button { + cursor: pointer; +} diff --git a/web/packages/dolphinScheduler/module/dispatch/dag/theme.scss b/web/packages/dolphinScheduler/module/dispatch/dag/theme.scss new file mode 100644 index 000000000..7694b66b9 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dag/theme.scss @@ -0,0 +1,27 @@ +@butterfly-theme-color-base:#fff; +@butterfly-normal-font-color-base:#222;; +@butterfly-overlay-font-color-base:#FFF; +@butterfly-primary-color-base:#F66902; +@butterfly-box-border-color-base: #D9D9D9; +@butterfly-box-shadow-base:0 2px 3px 0 rgba(0,112,204,0.06); +@butterfly-box-radius-base:100px; +@butterfly-line-color-base: #BFBFBF; +// 主题背景色(画布背景色) +@butterfly-theme-background-color: @butterfly-theme-color-base; +// (各图形框)普通情况背景色 +@butterfly-theme-color: fade(@butterfly-theme-color-base,80%); +// 主题色 +@butterfly-primary-color: @butterfly-primary-color-base; +// 普通情况下字体颜色 +@butterfly-normal-font-color: @butterfly-normal-font-color-base; +// 主题背景色上的字体颜色及icon颜色 +@butterfly-overlay-font-color: @butterfly-overlay-font-color-base; +// 拖动时的背景色 +@butterfly-box-move-background-color: fade(@butterfly-primary-color, 20%); +// 边框的样式 +@butterfly-box-border: 1px solid @butterfly-box-border-color-base; +@butterfly-box-shadow: @butterfly-box-shadow-base; +@butterfly-box-radius: @butterfly-box-radius-base; +@butterfly-box-node-border-hover: 1px solid @butterfly-primary-color; +// 线条样式 +@butterfly-line-color: @butterfly-line-color-base; diff --git a/web/packages/dolphinScheduler/module/dispatch/dashboard/index.vue b/web/packages/dolphinScheduler/module/dispatch/dashboard/index.vue new file mode 100644 index 000000000..1b641a66a --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dashboard/index.vue @@ -0,0 +1,712 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/dashboard/source/chartConfig.js b/web/packages/dolphinScheduler/module/dispatch/dashboard/source/chartConfig.js new file mode 100644 index 000000000..2ae48b052 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dashboard/source/chartConfig.js @@ -0,0 +1,81 @@ +import _ from 'lodash' +import { tasksState } from '../../config' + +const pie = { + series: [ + { + type: 'pie', + clickable: true, // Whether to open clicks + minAngle: 5, // The smallest sector angle (0 ~ 360), used to prevent a value from being too small, causing the sector to be too small to affect the interaction + avoidLabelOverlap: true, // Whether to prevent the label overlap policy + hoverAnimation: true, // Whether to enable hover to enlarge the animation on the sector. + radius: ['45%', '60%'], + center: ['50%', '50%'], + label: { + align: 'left', + normal: { + } + } + } + ] +} + +const bar = { + title: { + text: '' + }, + grid: { + right: '2%' + }, + xAxis: { + splitLine: { + show: false + }, + axisLabel: { + formatter (v) { + return `${v.split(',')[0]} (${v.split(',')[2]})` + } + } + }, + tooltip: { + formatter (v) { + const val = v[0].name.split(',') + return `${val[0]} (${v[0].value})` + } + }, + series: [{ + type: 'bar', + barWidth: 30 + }] +} + +const simple = { + xAxis: { + splitLine: { + show: false + }, + axisLabel: { + interval: 0, + showMaxLabel: true, + formatter (v) { + return tasksState[v].desc + } + } + }, + tooltip: { + formatter (data) { + let str = '' + _.map(data, (v, i) => { + if (i === 0) { + str += `${tasksState[v.name].desc}
` + } + str += `
${v.seriesName} : ${v.data}
` + }) + return str + } + }, + color: ['#D5050B', '#0398E1'] + +} + +export { pie, bar, simple } diff --git a/web/packages/dolphinScheduler/module/dispatch/dashboard/source/defineUserCount.vue b/web/packages/dolphinScheduler/module/dispatch/dashboard/source/defineUserCount.vue new file mode 100644 index 000000000..0c5899a40 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dashboard/source/defineUserCount.vue @@ -0,0 +1,83 @@ + + diff --git a/web/packages/dolphinScheduler/module/dispatch/dashboard/source/processStateCount.vue b/web/packages/dolphinScheduler/module/dispatch/dashboard/source/processStateCount.vue new file mode 100644 index 000000000..e251bb094 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dashboard/source/processStateCount.vue @@ -0,0 +1,143 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/dashboard/source/taskStatusCount.vue b/web/packages/dolphinScheduler/module/dispatch/dashboard/source/taskStatusCount.vue new file mode 100644 index 000000000..bee9c72e8 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/dashboard/source/taskStatusCount.vue @@ -0,0 +1,121 @@ + + diff --git a/web/packages/dolphinScheduler/module/dispatch/gantt/img/dag_bg.png b/web/packages/dolphinScheduler/module/dispatch/gantt/img/dag_bg.png new file mode 100644 index 000000000..d9fa09817 Binary files /dev/null and b/web/packages/dolphinScheduler/module/dispatch/gantt/img/dag_bg.png differ diff --git a/web/packages/dolphinScheduler/module/dispatch/gantt/index.js b/web/packages/dolphinScheduler/module/dispatch/gantt/index.js new file mode 100644 index 000000000..6e5a1b9f4 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/gantt/index.js @@ -0,0 +1,2 @@ +import gantt from './index.vue'; +export default gantt; diff --git a/web/packages/dolphinScheduler/module/dispatch/gantt/index.vue b/web/packages/dolphinScheduler/module/dispatch/gantt/index.vue new file mode 100644 index 000000000..c7b57d804 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/gantt/index.vue @@ -0,0 +1,195 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/gantt/source/gantt.js b/web/packages/dolphinScheduler/module/dispatch/gantt/source/gantt.js new file mode 100644 index 000000000..c23d56ca5 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/gantt/source/gantt.js @@ -0,0 +1,170 @@ +/*eslint-disable */ + +import _ from 'lodash' +import 'bootstrap/dist/css/bootstrap.min.css' +import 'bootstrap/dist/js/bootstrap.min.js' +import * as d3 from 'd3' +import { formatDate } from '../../convertor' +import { tasksState } from '../../config' + +const Gantt = function () { + this.el = '' + this.tasks = [] + this.width = null + this.height = null + this.taskNames = [] + this.tickFormat = '%H:%M:%S' + this.margin = { + top: 10, + right: 40, + bottom: 10, + left: 300 + } + this.startTimeXAxis = d3.time.day.offset(new Date(), -3) + this.endTimeXAxis = d3.time.hour.offset(new Date(), +3) +} + +Gantt.prototype.init = function ({ el, tasks }) { + this.el = el + this.tasks = tasks + this.taskNames = _.map(_.cloneDeep(tasks), v => v.taskName) + this.taskNames = this.taskNames.reduce(function (prev, cur) { + prev.indexOf(cur) === -1 && prev.push(cur) + return prev + }, []) + this.height = parseInt(this.taskNames.length * 30) + this.width = $(this.el).width() - this.margin.right - this.margin.left - 5 + + this.x = d3.time.scale() + .domain([this.startTimeXAxis, this.endTimeXAxis]) + .range([0, this.width]) + .clamp(true) + + this.y = d3.scale.ordinal() + .domain(this.taskNames) + .rangeRoundBands([0, this.height - this.margin.top - this.margin.bottom], 0.1) + + this.xAxis = d3.svg.axis() + .scale(this.x) + .orient('bottom') + .tickFormat(d3.time.format(this.tickFormat)) + .tickSubdivide(true) + .tickSize(8) + .tickPadding(8) + + this.yAxis = d3.svg.axis() + .scale(this.x) + .orient('left') + .tickSize(0) + + // time dimension + this.compXAxisTimes() + // Timescale calculation + this.initializeXAxis() + // Drawing charts + this.drawChart() +} + +/** + * Calculate time latitude + */ +Gantt.prototype.compXAxisTimes = function () { + if (this.tasks === undefined || this.tasks.length < 1) { + this.startTimeXAxis = d3.time.day.offset(new Date(), -3) + this.endTimeXAxis = d3.time.hour.offset(new Date(), +3) + return + } + this.tasks.sort((a, b) => a.endDate - b.endDate) + this.endTimeXAxis = this.tasks[this.tasks.length - 1].endDate + this.tasks.sort((a, b) => a.startDate - b.startDate) + this.startTimeXAxis = this.tasks[0].startDate +} + +/** + * Timescale processing + */ +Gantt.prototype.initializeXAxis = function () { + this.x = d3.time.scale() + .domain([this.startTimeXAxis, this.endTimeXAxis]) + .range([0, this.width]) + .clamp(true) + + this.y = d3.scale.ordinal() + .domain(this.taskNames) + .rangeRoundBands([0, this.height - this.margin.top - this.margin.bottom], 0.1) + + this.xAxis = d3.svg.axis() + .scale(this.x) + .orient('bottom') + .tickFormat(d3.time.format(this.tickFormat)) + .tickSubdivide(true) + .tickSize(8).tickPadding(8) + + this.yAxis = d3.svg.axis() + .scale(this.y) + .orient('left') + .tickSize(0) +} + +/** + * Drawing charts + */ +Gantt.prototype.drawChart = function () { + const svg = d3.select(this.el) + .append('svg') + .attr('class', 'chart') + .attr('width', this.width + this.margin.left + this.margin.right) + .attr('height', this.height + this.margin.top + this.margin.bottom + 150) + .append('g') + .attr('class', 'gantt-chart') + .attr('width', this.width + this.margin.left + this.margin.right) + .attr('height', this.height + this.margin.top + this.margin.bottom + 150) + .attr('transform', 'translate(' + this.margin.left + ', ' + this.margin.top + ')') + + svg.selectAll('.chart') + .data(this.tasks, d => d.startDate + d.taskName + d.endDate).enter() + .append('rect') + .attr('fill', d => d.status ? tasksState[d.status].color : '#000') + .attr('data-toggle', 'tooltip') + .attr('class', 'nodesp') + .attr('data-container', 'body') + .attr('title', this.tip) + .attr('y', 0) + .attr('transform', d => 'translate(' + this.x(d.startDate) + ',' + this.y(d.taskName) + ')') + .attr('height', () => this.y.rangeBand()) + .attr('width', d => d3.max([this.x(d.endDate) - this.x(d.startDate), 1])) + + svg.append('g') + .attr('class', 'x axis') + .attr('transform', 'translate(0, ' + (this.height - this.margin.top - this.margin.bottom) + ')') + .transition() + .call(this.xAxis) + .selectAll('text') + .attr('transform', `rotate(-${this.width / ($('.tick').length - 1) > 50 ? 0 : Math.acos(this.width / ($('.tick').length - 1) / 50) * 57})`) + .style('text-anchor', `${this.width / ($('.tick').length - 1) > 50 ? 'middle' : 'end'}`) + + svg.append('g') + .attr('class', 'y axis') + .transition() + .call(this.yAxis) + + $('rect.nodesp').tooltip({ + html: true, + container: 'body' + }) +} +/** + * Tip prompt + */ +Gantt.prototype.tip = function (d) { + let str = '
' + str += `taskName : ${d.taskName}
` + str += `status : ${tasksState[d.status].desc} (${d.status})
` + str += `startTime : ${formatDate(d.isoStart)}
` + str += `endTime : ${formatDate(d.isoEnd)}
` + str += `duration : ${d.duration}
` + str += '
' + return str +} + +export default new Gantt() diff --git a/web/packages/dolphinScheduler/module/dispatch/index.js b/web/packages/dolphinScheduler/module/dispatch/index.js new file mode 100644 index 000000000..0be153680 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/index.js @@ -0,0 +1,2 @@ +import index from './index.vue'; +export default index diff --git a/web/packages/dolphinScheduler/module/dispatch/index.vue b/web/packages/dolphinScheduler/module/dispatch/index.vue new file mode 100644 index 000000000..e94a6f86b --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/index.vue @@ -0,0 +1,1722 @@ + + + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/log/download.js b/web/packages/dolphinScheduler/module/dispatch/log/download.js new file mode 100644 index 000000000..59213c3c4 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/log/download.js @@ -0,0 +1,35 @@ +/** + * download file + */ +import api from '@dataspherestudio/shared/common/service/api'; + +const downloadFile = ($url, $obj, $fileName) => { + const downloadBlob = (data, fileNameS = 'json') => { + if (!data) { + return + } + const blob = new Blob([data]) + const fileName = `${fileNameS}` + if ('download' in document.createElement('a')) { // 不是IE浏览器 + const url = window.URL.createObjectURL(blob) + const link = document.createElement('a') + link.style.display = 'none' + link.href = url + link.setAttribute('download', fileName) + document.body.appendChild(link) + link.click() + document.body.removeChild(link) // 下载完成移除元素 + window.URL.revokeObjectURL(url) // 释放掉blob对象 + } else { // IE 10+ + window.navigator.msSaveBlob(blob, fileName) + } + } + api.fetch($url, $obj, { + method: 'get', + responseType: 'blob' + }).then((res) => { + downloadBlob(res.data, $fileName || `${Date.now()}.log`) + }) +} + +export { downloadFile } diff --git a/web/packages/dolphinScheduler/module/dispatch/log/log.vue b/web/packages/dolphinScheduler/module/dispatch/log/log.vue new file mode 100644 index 000000000..94ca42fce --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/log/log.vue @@ -0,0 +1,350 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/run.vue b/web/packages/dolphinScheduler/module/dispatch/run.vue new file mode 100644 index 000000000..593ea7c32 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/run.vue @@ -0,0 +1,447 @@ + + + + diff --git a/web/packages/dolphinScheduler/module/dispatch/timing.vue b/web/packages/dolphinScheduler/module/dispatch/timing.vue new file mode 100644 index 000000000..613c4b756 --- /dev/null +++ b/web/packages/dolphinScheduler/module/dispatch/timing.vue @@ -0,0 +1,538 @@ + + + + diff --git a/web/packages/dolphinScheduler/package.json b/web/packages/dolphinScheduler/package.json new file mode 100644 index 000000000..dba7f5d86 --- /dev/null +++ b/web/packages/dolphinScheduler/package.json @@ -0,0 +1,11 @@ +{ + "name": "@dataspherestudio/dolphinScheduler", + "version": "1.1.4", + "dependencies": { + "@dataspherestudio/shared": "^1.1.4", + "bootstrap": "^3.4.1", + "d3": "3.5.17", + "dayjs": "1.10.7", + "moment-timezone": "^0.5.34" + } +} diff --git a/web/packages/dolphinScheduler/router.js b/web/packages/dolphinScheduler/router.js new file mode 100644 index 000000000..f06c20470 --- /dev/null +++ b/web/packages/dolphinScheduler/router.js @@ -0,0 +1,23 @@ +export const subAppRoutes = { + path: '', + name: 'layout', + component: () => import('./view/layout.vue'), + redirect: '/scheduler', + meta: { + publicPage: true, // 权限公开 + }, + children: [] +} +export default [ + { + path: 'scheduler', + name: 'Scheduler', + meta: { + title: 'My Scheduler', + publicPage: true, + parent: 'Project', + }, + component: () => + import('./view/scheduler/index.vue'), + } +] diff --git a/web/packages/dolphinScheduler/view/layout.vue b/web/packages/dolphinScheduler/view/layout.vue new file mode 100644 index 000000000..e4e39b959 --- /dev/null +++ b/web/packages/dolphinScheduler/view/layout.vue @@ -0,0 +1,29 @@ + + + \ No newline at end of file diff --git a/web/packages/dolphinScheduler/view/scheduler/index.vue b/web/packages/dolphinScheduler/view/scheduler/index.vue new file mode 100644 index 000000000..da24340cc --- /dev/null +++ b/web/packages/dolphinScheduler/view/scheduler/index.vue @@ -0,0 +1,352 @@ + + + diff --git a/web/packages/dss/.env b/web/packages/dss/.env new file mode 100644 index 000000000..bbc42616e --- /dev/null +++ b/web/packages/dss/.env @@ -0,0 +1,4 @@ +VUE_APP_HOST= +VUE_APP_MN_CONFIG_PREFIX= +VUE_APP_MN_CONFIG_SOCKET=/ws/api/entrance/connect +NODE_ENV=production diff --git a/web/packages/dss/assets/images/2-1.svg b/web/packages/dss/assets/images/2-1.svg new file mode 100644 index 000000000..bf9d0e8d8 --- /dev/null +++ b/web/packages/dss/assets/images/2-1.svg @@ -0,0 +1,11 @@ + + + + 直线 6备份 3 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/2-2.svg b/web/packages/dss/assets/images/2-2.svg new file mode 100644 index 000000000..fe4bf07f5 --- /dev/null +++ b/web/packages/dss/assets/images/2-2.svg @@ -0,0 +1,11 @@ + + + + 直线 6备份 4 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/2-3.svg b/web/packages/dss/assets/images/2-3.svg new file mode 100644 index 000000000..a960d037d --- /dev/null +++ b/web/packages/dss/assets/images/2-3.svg @@ -0,0 +1,11 @@ + + + + 直线 6备份 6 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/3-1.svg b/web/packages/dss/assets/images/3-1.svg new file mode 100644 index 000000000..46b6fb42f --- /dev/null +++ b/web/packages/dss/assets/images/3-1.svg @@ -0,0 +1,11 @@ + + + + 直线 6备份 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/4-1.svg b/web/packages/dss/assets/images/4-1.svg new file mode 100644 index 000000000..778a72498 --- /dev/null +++ b/web/packages/dss/assets/images/4-1.svg @@ -0,0 +1,11 @@ + + + + 直线 7备份 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/5-1.svg b/web/packages/dss/assets/images/5-1.svg new file mode 100644 index 000000000..c804660ec --- /dev/null +++ b/web/packages/dss/assets/images/5-1.svg @@ -0,0 +1,11 @@ + + + + 直线 6备份 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/6-1.svg b/web/packages/dss/assets/images/6-1.svg new file mode 100644 index 000000000..106bff4c3 --- /dev/null +++ b/web/packages/dss/assets/images/6-1.svg @@ -0,0 +1,11 @@ + + + + 直线 6备份 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/7-1.svg b/web/packages/dss/assets/images/7-1.svg new file mode 100644 index 000000000..ded4154c1 --- /dev/null +++ b/web/packages/dss/assets/images/7-1.svg @@ -0,0 +1,11 @@ + + + + 直线 6 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/8-1.svg b/web/packages/dss/assets/images/8-1.svg new file mode 100644 index 000000000..7010dd8f3 --- /dev/null +++ b/web/packages/dss/assets/images/8-1.svg @@ -0,0 +1,11 @@ + + + + 直线 6 + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/appmap.png b/web/packages/dss/assets/images/appmap.png new file mode 100644 index 000000000..f96446c35 Binary files /dev/null and b/web/packages/dss/assets/images/appmap.png differ diff --git a/web/packages/dss/assets/images/bg8.png b/web/packages/dss/assets/images/bg8.png new file mode 100644 index 000000000..6b1f527d1 Binary files /dev/null and b/web/packages/dss/assets/images/bg8.png differ diff --git a/web/packages/dss/assets/images/dssLogo.png b/web/packages/dss/assets/images/dssLogo.png new file mode 100644 index 000000000..2619ce7d1 Binary files /dev/null and b/web/packages/dss/assets/images/dssLogo.png differ diff --git a/web/packages/dss/assets/images/handsome.jpg b/web/packages/dss/assets/images/handsome.jpg new file mode 100644 index 000000000..edcea88f5 Binary files /dev/null and b/web/packages/dss/assets/images/handsome.jpg differ diff --git a/web/packages/dss/assets/images/icon-nav-all.png b/web/packages/dss/assets/images/icon-nav-all.png new file mode 100644 index 000000000..d30aadcc1 Binary files /dev/null and b/web/packages/dss/assets/images/icon-nav-all.png differ diff --git a/web/packages/dss/assets/images/icon-nav-close.png b/web/packages/dss/assets/images/icon-nav-close.png new file mode 100644 index 000000000..ac9341c31 Binary files /dev/null and b/web/packages/dss/assets/images/icon-nav-close.png differ diff --git a/web/packages/dss/assets/images/icon-nav-din.png b/web/packages/dss/assets/images/icon-nav-din.png new file mode 100644 index 000000000..8c4be5dbf Binary files /dev/null and b/web/packages/dss/assets/images/icon-nav-din.png differ diff --git a/web/packages/dss/assets/images/icon-nav-ding.png b/web/packages/dss/assets/images/icon-nav-ding.png new file mode 100644 index 000000000..cd976d584 Binary files /dev/null and b/web/packages/dss/assets/images/icon-nav-ding.png differ diff --git a/web/packages/dss/assets/images/icon-nav-star.png b/web/packages/dss/assets/images/icon-nav-star.png new file mode 100644 index 000000000..8aa874ada Binary files /dev/null and b/web/packages/dss/assets/images/icon-nav-star.png differ diff --git a/web/packages/dss/assets/images/icon-nav-stard.png b/web/packages/dss/assets/images/icon-nav-stard.png new file mode 100644 index 000000000..fc0215511 Binary files /dev/null and b/web/packages/dss/assets/images/icon-nav-stard.png differ diff --git a/web/packages/dss/assets/images/login_bg.png b/web/packages/dss/assets/images/login_bg.png new file mode 100644 index 000000000..827992ab9 Binary files /dev/null and b/web/packages/dss/assets/images/login_bg.png differ diff --git a/web/packages/dss/assets/images/login_bg_small.png b/web/packages/dss/assets/images/login_bg_small.png new file mode 100644 index 000000000..931e36c73 Binary files /dev/null and b/web/packages/dss/assets/images/login_bg_small.png differ diff --git a/web/packages/dss/assets/images/loginbgc.svg b/web/packages/dss/assets/images/loginbgc.svg new file mode 100644 index 000000000..9cbbac10f --- /dev/null +++ b/web/packages/dss/assets/images/loginbgc.svg @@ -0,0 +1,1931 @@ + + + + 编组 7 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/luBanLogo.png b/web/packages/dss/assets/images/luBanLogo.png new file mode 100644 index 000000000..e759ee8b7 Binary files /dev/null and b/web/packages/dss/assets/images/luBanLogo.png differ diff --git a/web/packages/dss/assets/images/luban-menu-trigger.png b/web/packages/dss/assets/images/luban-menu-trigger.png new file mode 100644 index 000000000..cc479bd75 Binary files /dev/null and b/web/packages/dss/assets/images/luban-menu-trigger.png differ diff --git a/web/packages/dss/assets/images/newLogo.svg b/web/packages/dss/assets/images/newLogo.svg new file mode 100644 index 000000000..9b01dd191 --- /dev/null +++ b/web/packages/dss/assets/images/newLogo.svg @@ -0,0 +1,9 @@ + + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/images/no-data.svg b/web/packages/dss/assets/images/no-data.svg new file mode 100644 index 000000000..3b6861012 --- /dev/null +++ b/web/packages/dss/assets/images/no-data.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dss/assets/images/no_data.png b/web/packages/dss/assets/images/no_data.png new file mode 100644 index 000000000..5963a40cc Binary files /dev/null and b/web/packages/dss/assets/images/no_data.png differ diff --git a/web/packages/dss/assets/images/u1439.svg b/web/packages/dss/assets/images/u1439.svg new file mode 100644 index 000000000..5d7e56546 --- /dev/null +++ b/web/packages/dss/assets/images/u1439.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dss/assets/images/workspace-top.jpeg b/web/packages/dss/assets/images/workspace-top.jpeg new file mode 100644 index 000000000..bfc6f2097 Binary files /dev/null and b/web/packages/dss/assets/images/workspace-top.jpeg differ diff --git a/web/packages/dss/assets/images/workspace.png b/web/packages/dss/assets/images/workspace.png new file mode 100644 index 000000000..c2f502234 Binary files /dev/null and b/web/packages/dss/assets/images/workspace.png differ diff --git a/web/packages/dss/assets/images/workspace.svg b/web/packages/dss/assets/images/workspace.svg new file mode 100644 index 000000000..fcbfdf272 --- /dev/null +++ b/web/packages/dss/assets/images/workspace.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dss/assets/images/workspace1.svg b/web/packages/dss/assets/images/workspace1.svg new file mode 100644 index 000000000..529ecfec3 --- /dev/null +++ b/web/packages/dss/assets/images/workspace1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/dss/assets/projectIconFont/fonts/iconfont.eot b/web/packages/dss/assets/projectIconFont/fonts/iconfont.eot new file mode 100644 index 000000000..115ae2ceb Binary files /dev/null and b/web/packages/dss/assets/projectIconFont/fonts/iconfont.eot differ diff --git a/web/packages/dss/assets/projectIconFont/fonts/iconfont.svg b/web/packages/dss/assets/projectIconFont/fonts/iconfont.svg new file mode 100644 index 000000000..e106c6103 --- /dev/null +++ b/web/packages/dss/assets/projectIconFont/fonts/iconfont.svg @@ -0,0 +1,37 @@ + + + +Generated by IcoMoon + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/web/packages/dss/assets/projectIconFont/fonts/iconfont.ttf b/web/packages/dss/assets/projectIconFont/fonts/iconfont.ttf new file mode 100644 index 000000000..b0b5b879f Binary files /dev/null and b/web/packages/dss/assets/projectIconFont/fonts/iconfont.ttf differ diff --git a/web/packages/dss/assets/projectIconFont/fonts/iconfont.woff b/web/packages/dss/assets/projectIconFont/fonts/iconfont.woff new file mode 100644 index 000000000..2f3711802 Binary files /dev/null and b/web/packages/dss/assets/projectIconFont/fonts/iconfont.woff differ diff --git a/web/packages/dss/assets/projectIconFont/readme.md b/web/packages/dss/assets/projectIconFont/readme.md new file mode 100644 index 000000000..d0924847c --- /dev/null +++ b/web/packages/dss/assets/projectIconFont/readme.md @@ -0,0 +1,3 @@ +增加图标去icomoon,导入selection.json,添加需要的图标之后再导出。 + +https://icomoon.io/app/#/select/f \ No newline at end of file diff --git a/web/packages/dss/assets/projectIconFont/selection.json b/web/packages/dss/assets/projectIconFont/selection.json new file mode 100644 index 000000000..d44468e57 --- /dev/null +++ b/web/packages/dss/assets/projectIconFont/selection.json @@ -0,0 +1 @@ +{"IcoMoonType":"selection","icons":[{"icon":{"paths":["M940.8 883.2h-860.8v-739.2h860.8zM112 851.2h796.8v-675.2h-796.8z","M194.464 618.112l-17.76-26.624 155.008-103.328-156.384-130.272 20.512-24.576 189.28 157.696-190.656 127.104zM406.4 582.4h249.6v32h-249.6zM80 176h860.8v60.8h-860.8z"],"attrs":[{},{}],"isMulticolor":false,"isMulticolor2":false,"grid":0,"tags":["fi-scriptis"]},"attrs":[{},{}],"properties":{"order":35,"id":0,"name":"scriptis","prevSize":32,"code":59648},"setIdx":0,"setId":6,"iconIdx":0},{"icon":{"paths":["M870.4 792.576h-272.384v96.256h77.824v57.344h-327.68v-57.344h77.824v-96.256h-272.384c-36.864 0-77.824-32.768-77.824-69.632v-579.584c0-36.864 30.72-67.584 67.584-67.584h735.232c36.864 0 69.632 30.72 69.632 67.584v577.536c0 38.912-40.96 71.68-77.824 71.68zM888.832 174.080c0-24.576-14.336-38.912-38.912-38.912h-675.84c-24.576 0-38.912 14.336-38.912 38.912v444.416c0 24.576 14.336 38.912 38.912 38.912h677.888c24.576 0 38.912-14.336 38.912-38.912l-0-444.416zM765.952 348.16c-2.048 6.144-4.096 12.288-6.144 16.384 0 0 16.384 20.48 6.144 30.72l-4.096 4.096c-8.192 8.192-24.576-4.096-28.672-6.144-6.144 4.096-12.288 6.144-18.432 6.144h2.048c0 0-4.096 26.624-16.384 26.624h-4.096c-10.24 0-16.384-22.528-16.384-26.624-6.144-2.048-12.288-4.096-18.432-8.192v0c0 0-20.48 16.384-30.72 6.144l-4.096-4.096c-8.192-8.192 4.096-26.624 6.144-30.72-2.048-4.096-6.144-10.24-6.144-16.384-4.096 0-26.624-6.144-26.624-16.384v-4.096c0-12.288 20.48-16.384 26.624-16.384 2.048-6.144 4.096-12.288 6.144-16.384-2.048-4.096-14.336-22.528-8.192-30.72l4.096-4.096c8.192-8.192 26.624 4.096 30.72 6.144 6.144-2.048 10.24-6.144 16.384-6.144 2.048-6.144 6.144-26.624 16.384-26.624h4.096c12.288 0 16.384 20.48 16.384 26.624 6.144 2.048 12.288 4.096 16.384 6.144 4.096-2.048 22.528-14.336 30.72-6.144l4.096 4.096c8.192 8.192-4.096 24.576-6.144 30.72 2.048 6.144 6.144 10.24 6.144 16.384 2.048 0 24.576 4.096 24.576 16.384v4.096c4.096 10.24-16.384 16.384-22.528 18.432zM696.32 274.432c-30.72 0-55.296 24.576-55.296 55.296s24.576 55.296 55.296 55.296 55.296-24.576 55.296-55.296-26.624-55.296-55.296-55.296zM696.32 364.544c-20.48 0-36.864-16.384-36.864-36.864s16.384-36.864 36.864-36.864c20.48 0 36.864 16.384 36.864 36.864-2.048 20.48-18.432 36.864-36.864 36.864zM696.32 309.248c-10.24 0-18.432 8.192-18.432 18.432s8.192 18.432 18.432 18.432c10.24 0 18.432-8.192 18.432-18.432s-8.192-18.432-18.432-18.432zM550.912 432.128c-4.096 10.24-8.192 22.528-12.288 30.72 0 2.048 30.72 38.912 12.288 57.344l-8.192 8.192c-14.336 14.336-47.104-8.192-55.296-12.288-10.24 6.144-22.528 10.24-32.768 14.336h2.048c0 0-6.144 49.152-32.768 49.152h-8.192c-20.48 0-30.72-40.96-32.768-49.152-12.288-4.096-22.528-8.192-32.768-14.336l2.048 2.048c0 0-38.912 30.72-57.344 12.288l-8.192-6.144c-14.336-14.336 8.192-51.2 12.288-59.392-6.144-10.24-10.24-20.48-12.288-30.72-8.192-2.048-49.152-12.288-49.152-32.768v-8.192c0-22.528 40.96-30.72 49.152-32.768 2.048-10.24 8.192-20.48 12.288-30.72-4.096-6.144-28.672-43.008-14.336-57.344l8.192-8.192c16.384-16.384 49.152 6.144 57.344 12.288 10.24-6.144 20.48-10.24 30.72-12.288 2.048-10.24 14.336-49.152 32.768-49.152h8.192c22.528 0 30.72 36.864 32.768 49.152 12.288 4.096 22.528 8.192 32.768 12.288 8.192-6.144 43.008-26.624 57.344-12.288l8.192 8.192c16.384 16.384-6.144 49.152-12.288 57.344 6.144 10.24 10.24 20.48 12.288 32.768 2.048 0 49.152 8.192 49.152 32.768v8.192c-4.096 16.384-40.96 26.624-51.2 28.672zM415.744 292.864c-57.344 0-102.4 47.104-102.4 102.4 0 57.344 47.104 102.4 102.4 102.4 57.344 0 102.4-47.104 102.4-102.4s-45.056-102.4-102.4-102.4zM415.744 464.896c-38.912 0-69.632-30.72-69.632-69.632s30.72-69.632 69.632-69.632c38.912 0 69.632 30.72 69.632 69.632-2.048 38.912-32.768 69.632-69.632 69.632zM415.744 362.496c-18.432 0-34.816 16.384-34.816 34.816s16.384 34.816 34.816 34.816c18.432 0 34.816-16.384 34.816-34.816-0-20.48-16.384-34.816-34.816-34.816z"],"isMulticolor":false,"isMulticolor2":false,"tags":["guanlishitu"],"defaultCode":58888,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":1,"order":9,"ligatures":"","prevSize":32,"code":58888,"name":"guanlishitu"},"setIdx":1,"setId":5,"iconIdx":0},{"icon":{"paths":["M410.5 452.2c191.9 0 346.3-64.9 346.3-144.3 0-14.4-4.3-27.4-14.4-40.4-4.3-5.8-8.7-10.1-13-15.9-2.9-2.9-5.8-5.8-8.7-7.2-2.9-2.9-7.2-5.8-11.5-8.7-2.9-1.4-5.8-4.3-8.7-5.8-5.8-4.3-13-7.2-18.8-11.5-13-7.2-28.9-13-44.7-20.2-15.9-5.8-33.2-11.5-51.9-15.9-11.5-2.9-24.5-5.8-37.5-8.7-31.7-5.8-66.4-10.1-102.4-11.5-11.5 1.4-23.1 1.4-34.6 1.4s-23.1 0-36.1 1.4c-36.1 1.4-70.7 5.8-102.4 11.5-13 2.9-26 4.3-37.5 7.2-17.3 4.3-34.6 8.7-50.5 14.4-15.9 5.8-28.9 11.5-41.8 17.3-4.3 2.9-10.1 5.8-14.4 8.7-14.4 8.7-26 17.3-36.1 27.4-7.2 7.2-13 14.4-17.3 21.6-7.2 11.5-10.1 23.1-10.1 34.6-0.2 79.6 154.2 144.6 346.1 144.6zM756.8 509.9v-92.3c-75 60.6-210.7 92.3-346.3 92.3s-271.3-31.7-346.3-92.3v92.3c0 71.8 126.2 131.7 292.3 142.5v-102.5c0-5.6 4.6-10.2 10.2-10.2h382.7c4.8-9.6 7.4-19.6 7.4-29.8zM165.2 567.6c-15.9 0-28.9-13-28.9-28.9s13-28.9 28.9-28.9c15.9 0 28.9 13 28.9 28.9s-13 28.9-28.9 28.9zM64.2 726.3h1.4c15.6 66.1 135.8 118.2 290.8 128.1v-144.2c-116.7-7.3-227.2-38.1-292.3-90.7v106.8zM165.2 711.9c15.9 0 28.9 13 28.9 28.9s-13 28.9-28.9 28.9c-15.9 0-28.9-13-28.9-28.9 0.1-15.9 13-28.9 28.9-28.9zM462.1 793.5c0 6.5-0.8 11.8-2.5 15.8s-3.9 7.1-6.5 9.2c-2.7 2.4-5.4 4-8.3 4.9-2.9 0.8-5.4 1.3-7.6 1.3-3.9 0-7.6-0.9-11.1-2.7s-6.6-4-9.2-6.7l-29 23.2c4.6 5.1 10.8 10.2 18.7 15.4 7.8 5.2 17.7 7.8 29.5 7.8 10.1 0 19-1.7 26.6-5.1s14.2-8 19.7-13.8c11.1-11.8 16.7-26.6 16.7-44.2v-197.1h-37v192zM588.6 601.5h-54.7v258h51.4c22 0 38.8-5.7 50.5-17.2s17.6-29.3 17.6-53.4v-120.7c0-21.3-5.5-37.7-16.5-49.3s-27.1-17.4-48.3-17.4zM616.5 792.4c0 11.1-2.2 19.3-6.5 24.5s-11.7 7.8-22.1 7.8h-17v-188.4h17c10.9 0 18.4 2.8 22.5 8.3 4.1 5.6 6.2 13.4 6.2 23.6v124.2zM792.9 621.4c-5.3-6-12.1-10.9-20.5-14.5-8.3-3.6-18.4-5.4-30.3-5.4h-54v258h50.4c24.2 0 41.7-6 52.7-18.1s16.5-29.5 16.5-52.2v-14.5c0-13.8-2.1-24.6-6.2-32.6s-10.5-14-19.2-18.1c8.5-4.8 14.7-10.6 18.8-17.4 2.2-3.6 3.8-7.8 4.9-12.7 1.1-4.8 1.6-10.7 1.6-17.8v-9.4c0-8.5-1.1-16.6-3.4-24.5-2.2-7.8-6-14.7-11.3-20.8zM772.3 796.4c-0.5 5.3-1.8 9.9-4 13.8s-5.4 6.9-9.6 9.1c-4.2 2.2-10.1 3.3-17.6 3.3h-15.9v-82.6h16.3c6.8 0 12.2 0.8 16.3 2.4s7.4 4 9.8 7.4c2.2 3.4 3.6 7.5 4.3 12.3s1.1 10.5 1.1 17c0 6.2-0.3 12-0.7 17.3zM766.6 698c-4.2 6.2-12.5 9.2-24.8 9.2h-16.7v-71h15.2c12.3 0 20.8 2.9 25.5 8.7s7.1 14.5 7.1 26.1c0.1 11.9-2 20.9-6.3 27zM959.9 675.4v-14.5c0-8.9-1.5-17.2-4.5-24.8s-7.2-14.2-12.5-19.7c-10.9-11.4-25-17-42.4-17-8.9 0-17.1 1.5-24.5 4.5s-13.7 7.2-19 12.5c-5.3 5.3-9.4 11.6-12.3 18.8s-4.3 15.1-4.3 23.6v142c0 11.8 1.9 21.6 5.6 29.2s8.5 13.8 14.3 18.7c5.8 4.8 12.3 8.2 19.6 10.1s14.4 2.9 21.4 2.9c8.2 0 15.9-1.6 23-4.9s13.3-7.7 18.7-13.2c5.3-5.6 9.5-11.9 12.5-19s4.5-14.7 4.5-22.6v-15.9h-37v12.7c0 5.1-0.7 9.3-2.2 12.7-1.4 3.4-3.3 6-5.4 8-2.4 1.9-5 3.3-7.8 4.2-2.8 0.8-5.4 1.3-7.8 1.3-8.5 0-14.3-2.5-17.6-7.6s-4.9-11.4-4.9-18.8v-132.3c0-8.7 1.5-15.9 4.5-21.6s9.1-8.5 18.3-8.5c7 0 12.6 2.7 16.7 8s6.2 11.6 6.2 18.8v12.3h36.9z"],"isMulticolor":false,"isMulticolor2":false,"tags":["JDBC"],"defaultCode":59230,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":2,"order":10,"ligatures":"","prevSize":32,"code":59230,"name":"JDBC"},"setIdx":1,"setId":5,"iconIdx":1},{"icon":{"paths":["M213.844 203.564l596.4-87.928v197.952l-596.4 87.928zM213.788 457.312l596.4-87.928v197.952l-596.4 87.928zM213.768 710.4l596.392-87.932v197.952l-596.392 87.928zM225.12 366.636l382.416 123.684-11.328 35.028-382.416-123.684zM427.76 245.496l382.416 123.688-11.328 35.028-382.416-123.688zM225.112 620.148l382.416 123.684-11.328 35.032-382.416-123.688zM427.792 498.604l382.416 123.688-11.332 35.028-382.412-123.688z"],"isMulticolor":false,"isMulticolor2":false,"tags":["scala"],"defaultCode":59175,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":3,"order":11,"ligatures":"","prevSize":32,"code":59175,"name":"scala"},"setIdx":1,"setId":5,"iconIdx":2},{"icon":{"paths":["M820.304 663.769h-8.529l2.007-49.168c80.274 3.010 140.48-49.67 158.542-103.855 14.55-43.147 3.010-85.793-34.117-109.876-107.367-70.24-237.813 40.639-239.318 41.642l-33.615-35.622c1.505-1.505 38.632-36.123 92.817-59.704 74.756-32.11 146.501-30.605 207.208 11.038 55.189 38.13 75.257 102.852 53.683 167.573-24.082 69.738-98.838 137.972-198.679 137.972zM511.749 387.324l-47.663-10.034c1.505-6.522 36.625-165.064 176.102-173.092 87.298-5.017 131.951 42.144 143.992 70.24l-44.653 19.567c0-0.502-20.57-45.154-96.329-41.141-101.848 6.021-129.944 129.442-131.449 134.46zM219.751 157.037c0.502-2.509 0.502-5.017 0.502-7.526 0-29.099-23.581-53.182-53.182-53.182-29.099 0-53.182 23.581-53.182 53.182 0 29.099 23.581 53.182 53.182 53.182 15.051 0 28.096-6.021 37.629-16.055 24.082 14.55 21.072 39.134 20.069 44.653l32.11 6.522c4.515-20.57 0.502-59.704-37.127-80.776zM165.064 174.597c-13.546 0-24.584-11.038-24.584-24.584s11.038-24.584 24.584-24.584 24.584 11.038 24.584 24.584-11.038 24.584-24.584 24.584zM375.784 112.384c0-29.099-23.581-53.182-53.182-53.182s-53.182 23.581-53.182 53.182c0 29.099 23.581 53.182 53.182 53.182 3.512 0 7.024-0.502 10.034-1.003 3.512 11.038 9.031 42.144-28.096 78.268l22.577 23.581c55.189-53.683 40.639-102.852 34.117-117.903 9.031-9.533 14.55-22.075 14.55-36.123zM323.104 134.961c-13.546 0-24.584-11.038-24.584-24.584s11.038-24.584 24.584-24.584 24.584 11.038 24.584 24.584-11.038 24.584-24.584 24.584zM749.562 786.689l-68.233 95.326 186.638 76.261zM550.883 932.688c-52.178 0-104.858-15.051-152.020-43.649-94.824-57.697-149.009-160.047-136.968-260.39l48.666 5.519c-9.533 81.78 35.12 165.566 113.889 213.229 51.677 31.106 110.879 43.147 166.569 32.611 54.687-10.034 99.841-40.137 126.934-84.288s32.611-97.834 16.055-151.518c-16.557-54.185-54.185-101.346-105.862-132.954-53.683-32.611-116.9-43.649-174.095-31.106l-10.536-47.663c69.237-15.051 145.999-1.505 209.717 37.127 61.711 37.629 107.367 94.322 127.436 160.047 20.57 66.728 13.045 134.46-21.072 191.153-34.117 56.192-90.81 94.322-159.545 106.865-16.557 3.512-33.113 5.017-49.168 5.017zM290.493 751.569l-12.543-55.69c144.494-32.611 219.751-250.857 220.755-252.864l54.185 18.563c-3.512 10.034-86.797 250.356-262.397 289.991zM362.238 849.905l-15.553-54.687c246.844-70.24 262.397-298.52 262.397-301.029l57.195 3.010c0 3.010-4.014 69.237-43.649 145.999-36.625 70.742-111.381 164.061-260.39 206.707zM485.66 924.159l-18.062-54.185c206.205-69.237 239.318-288.486 239.318-290.493l46.659 7.024c-0.502 2.509-3.010 69.237-40.137 140.982-49.67 95.326-128.439 163.559-227.779 196.672zM244.837 222.26v40.137c68.735 0 109.374 54.185 124.425 104.858 17.56 58.701 7.526 136.466-54.185 177.105-23.079 15.051-46.659 22.577-70.742 22.577-60.707 0-112.384-46.158-137.972-73.752-41.141-44.653-62.213-90.81-63.718-108.37-0.502-8.529 12.041-32.611 62.714-66.226 36.625-24.584 74.756-41.642 79.773-43.649 20.57-8.027 40.639-12.041 59.704-12.041v-40.639M244.837 222.26c-23.079 0-48.165 4.515-75.257 15.051 0 0-172.59 74.756-167.071 150.013 4.515 61.209 112.384 219.751 241.325 219.751 30.103 0 61.711-8.529 92.817-29.099 146.501-95.828 80.274-355.716-91.814-355.716zM134.961 365.248c0 13.577 11.007 24.584 24.584 24.584s24.584-11.007 24.584-24.584h0c-0-13.577-11.007-24.584-24.584-24.584s-24.584 11.007-24.584 24.584v0z"],"width":1027,"isMulticolor":false,"isMulticolor2":false,"tags":["hive"],"defaultCode":58881,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":4,"order":12,"ligatures":"","prevSize":32,"code":58881,"name":"hive"},"setIdx":1,"setId":5,"iconIdx":3},{"icon":{"paths":["M732.96 757.393l198.75 58.18-118.625-244.331 179.994-230.806-269.497 67.494-132.149-228.605-53.077 285.222-268.445 95.787 197.697 89.28-134.222 94.702-159.198-70.747s-87.398-41.37-87.398-113.202 89.471-96.871 89.471-96.871l265.319-86.377 44.401-269.976s20.797-107.429 91.576-107.429 99.901 85.643 135.977 142.26l73.522 129.183 263.596-72.566s216.421-47.909 87.398 145.131l-187.299 229.339 111.001 226.436s86.026 184.332-116.551 159.676l-191.445-63.858-20.797-137.89zM519.378 521.77c-48.707 9.059-57.574 27.942-57.574 62.901v46.059h115.212v15.406h-158.432c-33.524 0-62.837 21.148-71.991 61.306-10.558 46.059-11.004 74.766 0 122.867 8.198 35.756 27.687 61.306 61.21 61.306h39.552v-55.214c0-39.935 32.854-75.117 71.991-75.117h115.116c32.025 0 57.574-27.718 57.574-61.434v-115.307c0-32.822-26.315-57.351-57.574-62.901-38.595-6.666-80.508-6.347-115.116 0.096zM512.169 553.89c11.866 0 21.658 10.398 21.658 23.061s-9.697 22.966-21.658 22.966c-11.961 0-21.658-10.303-21.658-22.966 0.096-12.791 9.697-23.061 21.658-23.061zM711.78 645.977v53.746c0 41.626-33.619 76.712-71.991 76.712h-115.116c-31.482 0-57.574 28.293-57.574 61.434v115.18c0 32.822 27.176 52.056 57.574 61.434 36.426 11.196 71.449 13.237 115.116 0 28.994-8.835 57.542-26.602 57.542-61.434v-46.059h-114.988v-15.406h172.658c33.524 0 45.932-24.561 57.574-61.306 12.057-37.894 11.547-74.32 0-122.867-8.293-34.959-24.018-61.306-57.574-61.306h-43.22v-0.096zM646.998 937.547c11.961 0 21.658 10.303 21.658 22.966 0 12.791-9.697 23.061-21.658 23.061-11.866 0-21.658-10.398-21.658-23.061 0.096-12.791 9.824-22.966 21.658-22.966z"],"width":1275,"isMulticolor":false,"isMulticolor2":false,"tags":["spark-python"],"defaultCode":58883,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":5,"order":13,"ligatures":"","prevSize":32,"code":58883,"name":"spark-python"},"setIdx":1,"setId":5,"iconIdx":4},{"icon":{"paths":["M953.927 0.002h-737.571c-119.579 0.416-216.356 97.449-216.356 217.086 0 0.078 0 0.155 0 0.233l-0-0.012v566.636c-0 0.044-0 0.096-0 0.148 0 119.637 96.777 216.671 216.317 217.086l0.040 0h737.571c119.579-0.416 216.356-97.449 216.356-217.086 0-0.078-0-0.155-0-0.233l0 0.012v-566.563c0.001-0.283 0.002-0.617 0.002-0.952 0-59.709-24.114-113.786-63.134-153.023l0.010 0.010c-38.916-39.128-92.792-63.344-152.324-63.344-0.32 0-0.64 0.001-0.959 0.002l0.049-0zM216.356 85.433h737.571c45.495 0 87.625 23.991 110.957 63.195l-425.471 347.428c-14.722 12.152-33.782 19.522-54.564 19.522s-39.843-7.37-54.711-19.639l0.146 0.117-425.471-347.428c23.991-38.766 66.048-62.61 111.543-63.195zM953.927 915.968h-737.571c-71.555-0.374-129.422-58.446-129.463-130.044l-0-0.004v-536.648l390.070 317.586c30.939 25.088 69.412 38.766 109.129 38.985 0.102 0 0.223 0 0.344 0 41.396 0 79.354-14.721 108.923-39.214l-0.285 0.229 390.143-317.659v536.867c-1.739 71.503-59.662 128.914-131.203 129.827l-0.088 0.001z"],"width":1170,"isMulticolor":false,"isMulticolor2":false,"tags":["mail"],"defaultCode":58893,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":6,"order":14,"ligatures":"","prevSize":32,"code":58893,"name":"mail"},"setIdx":1,"setId":5,"iconIdx":5},{"icon":{"paths":["M413.417 116.73c28.788 0 46.061 17.465 51.818 46.541 0 23.254-23.030 46.509-46.061 46.509s-46.061-23.254-46.061-46.509c-5.79-23.254 17.273-46.541 40.303-46.541zM984.251 391.592c22.231 95.128 22.231 173.496-5.534 246.233-22.231 67.172-49.995 134.344-138.886 134.344h-88.891v89.531c0 50.379-22.199 111.953-133.32 145.539-44.429 11.163-83.325 16.761-122.189 16.761-38.896 0-77.792-5.598-122.221-16.793-83.325-27.988-133.32-78.367-133.32-145.507v-89.563h-49.995c-77.76 0-127.755-50.347-149.985-139.91-22.231-89.531-22.231-151.105 0-246.233 16.665-83.965 77.76-134.344 166.651-139.91h61.095v-89.563c0-72.738 22.231-123.117 138.886-145.507 88.891-16.793 161.085-11.195 238.876 0 83.325 11.195 138.854 72.77 138.854 145.539v95.128h61.095c66.692 0 116.687 50.347 138.886 139.91zM239.889 637.825c0-89.563 77.76-167.898 172.216-173.496h222.179c49.995 0 88.891-39.152 88.891-89.531v-218.277c0-50.347-44.461-83.933-88.891-89.563-72.194-11.163-144.42-11.163-222.179 0-83.325 11.195-88.891 39.216-88.891 89.563v89.563h205.546c16.633 0 27.764 11.195 27.764 27.988s-11.099 27.956-27.764 27.956h-316.668c-33.298 0-99.958 16.793-116.623 95.16-22.231 89.563-22.231 139.91 0 223.843 16.633 61.574 44.429 95.16 94.425 95.16h49.995v-78.367zM928.722 621.032c22.199-61.542 22.199-128.714 5.534-218.245-22.199-83.965-61.095-95.16-88.859-95.16h-61.095v72.77c0 78.367-66.692 145.539-144.452 145.539h-227.745c-61.095 0-116.655 55.945-116.655 117.487v223.875c0 50.379 49.995 78.367 94.425 89.563 72.226 22.391 138.886 22.391 211.112 0 61.095-16.793 94.425-50.379 94.425-89.563v-89.563h-199.981c-16.665 0-27.796-11.195-27.796-27.956 0-16.793 11.131-27.988 27.796-27.988h344.4c44.461 0 61.095-22.391 88.891-100.758zM628.080 907.697c-23.254 0-46.509-23.286-46.509-46.541s23.254-46.509 46.509-46.509 46.541 23.254 46.541 46.509-23.286 46.541-46.541 46.541z"],"isMulticolor":false,"isMulticolor2":false,"tags":["python-o"],"defaultCode":58916,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":7,"order":15,"ligatures":"","prevSize":32,"code":58916,"name":"python-o"},"setIdx":1,"setId":5,"iconIdx":6},{"icon":{"paths":["M512 0c-215.808 0-448 65.056-448 208v608c0 142.88 232.192 208 448 208s448-65.12 448-208v-608c0-142.944-232.256-208-448-208zM896 816c0 79.488-171.936 144-384 144-212.096 0-384-64.512-384-144v-119.552c66.112 68.128 225.6 103.552 384 103.552s317.888-35.424 384-103.552v119.552zM896 624h-0.128c0 0.32 0.128 0.672 0.128 0.992 0 79.008-171.936 143.008-384 143.008s-384-64-384-143.008c0-0.32 0.128-0.672 0.128-0.992h-0.128v-119.552c66.112 68.128 225.6 103.552 384 103.552s317.888-35.424 384-103.552v119.552zM896 432h-0.128c0 0.32 0.128 0.672 0.128 0.992 0 79.008-171.936 143.008-384 143.008s-384-64-384-143.008c0-0.32 0.128-0.672 0.128-0.992h-0.128v-109.952c83.872 63.904 237.6 93.952 384 93.952s300.128-30.048 384-93.952v109.952zM512 352c-212.096 0-384-64.512-384-144 0-79.552 171.904-144 384-144 212.064 0 384 64.448 384 144 0 79.488-171.936 144-384 144zM768 832c0-17.664 14.336-32 32-32s32 14.336 32 32c0 17.664-14.336 32-32 32s-32-14.336-32-32zM768 640c0-17.664 14.336-32 32-32s32 14.336 32 32c0 17.664-14.336 32-32 32s-32-14.336-32-32zM768 448c0-17.664 14.336-32 32-32s32 14.336 32 32c0 17.664-14.336 32-32 32s-32-14.336-32-32z"],"isMulticolor":false,"isMulticolor2":false,"tags":["data"],"defaultCode":59051,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":8,"order":16,"ligatures":"","prevSize":32,"code":59051,"name":"data"},"setIdx":1,"setId":5,"iconIdx":7},{"icon":{"paths":["M851.2 0h-563.2c-76.8 0-134.4 57.6-134.4 134.4v19.2c-64 12.8-108.8 70.4-108.8 140.8 0 44.8 19.2 83.2 51.2 108.8-32 25.6-51.2 64-51.2 108.8s19.2 83.2 51.2 108.8c-32 25.6-51.2 64-51.2 108.8 0 70.4 44.8 121.6 108.8 140.8v19.2c0 70.4 57.6 134.4 134.4 134.4h563.2c70.4 0 134.4-57.6 134.4-134.4v-755.2c-6.4-76.8-64-134.4-134.4-134.4zM153.6 800c-25.6-12.8-44.8-38.4-44.8-70.4s19.2-57.6 44.8-70.4v140.8zM153.6 582.4c-25.6-12.8-44.8-38.4-44.8-70.4s19.2-57.6 44.8-70.4v140.8zM153.6 358.4c-25.6-6.4-44.8-38.4-44.8-64 0-32 19.2-57.6 44.8-70.4v134.4zM812.8 793.6h-460.8v-64h460.8v64zM812.8 563.2h-460.8v-64h460.8v64zM812.8 294.4h-460.8v-64h460.8v64z"],"isMulticolor":false,"isMulticolor2":false,"tags":["log"],"defaultCode":58960,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":9,"order":17,"ligatures":"","prevSize":32,"code":58960,"name":"log"},"setIdx":1,"setId":5,"iconIdx":8},{"icon":{"paths":["M853.333 1024h-682.667c-93.867 0-170.667-76.8-170.667-170.667v-640c0-25.6 17.067-42.667 42.667-42.667h938.667c25.6 0 42.667 17.067 42.667 42.667v640c-0 93.867-76.8 170.667-170.667 170.667zM85.333 256v597.333c0 46.933 38.4 85.333 85.333 85.333h682.667c46.933 0 85.333-38.4 85.333-85.333l0-597.333h-853.333zM981.333 213.333h-938.667v-42.667c0-72.533 55.467-128 128-128h682.667c72.533 0 128 55.467 128 128l-0 42.667zM981.333 256h-938.667c-25.6 0-42.667-17.067-42.667-42.667v-42.667c0-93.867 76.8-170.667 170.667-170.667h682.667c93.867 0 170.667 76.8 170.667 170.667v42.667c-0 25.6-17.067 42.667-42.667 42.667zM85.333 170.667h853.333c0-46.933-38.4-85.333-85.333-85.333l-682.667-0c-46.933 0-85.333 38.4-85.333 85.333zM298.667 1024c-25.6 0-42.667-17.067-42.667-42.667v-768c0-25.6 17.067-42.667 42.667-42.667s42.667 17.067 42.667 42.667v768c-0 25.6-17.067 42.667-42.667 42.667zM981.333 512h-938.667c-25.6 0-42.667-17.067-42.667-42.667s17.067-42.667 42.667-42.667h938.667c25.6 0 42.667 17.067 42.667 42.667s-17.067 42.667-42.667 42.667zM981.333 768h-938.667c-25.6 0-42.667-17.067-42.667-42.667s17.067-42.667 42.667-42.667h938.667c25.6 0 42.667 17.067 42.667 42.667s-17.067 42.667-42.667 42.667z"],"isMulticolor":false,"isMulticolor2":false,"tags":["table"],"defaultCode":59733,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":10,"order":18,"ligatures":"","prevSize":32,"code":59733,"name":"table"},"setIdx":1,"setId":5,"iconIdx":9},{"icon":{"paths":["M827.077 39.385c66.954 0 118.154 51.2 118.154 118.154v708.923c0 66.954-51.2 118.154-118.154 118.154h-630.154c-66.954 0-118.154-51.2-118.154-118.154v-708.923c0-66.954 51.2-118.154 118.154-118.154h630.154M827.077 0h-630.154c-86.646 0-157.538 70.892-157.538 157.538v708.923c0 86.646 70.892 157.538 157.538 157.538h630.154c86.646 0 157.538-70.892 157.538-157.538v-708.923c0-86.646-70.892-157.538-157.538-157.538zM512 196.923h393.846v39.385h-393.846zM512 393.846h393.846v39.385h-393.846zM512 590.769h393.846v39.385h-393.846zM118.154 787.692h787.692v39.385h-787.692zM259.938 630.154v-346.585h-129.969v-47.262h311.138v47.262h-126.031v346.585h-55.138z"],"isMulticolor":false,"isMulticolor2":false,"tags":["field"],"defaultCode":58898,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":11,"order":19,"ligatures":"","prevSize":32,"code":58898,"name":"field"},"setIdx":1,"setId":5,"iconIdx":10},{"icon":{"paths":["M792.32 960h-560.64c-49.92 0-90.432-41.792-90.432-93.312v-709.376c0-51.52 40.512-93.312 90.432-93.312h560.64c49.92 0 90.432 41.792 90.432 93.312v709.312c0 51.584-40.512 93.376-90.432 93.376zM233.92 820.992c0 25.591 20.745 46.336 46.336 46.336s46.336-20.745 46.336-46.336v-0c0-25.591-20.745-46.336-46.336-46.336s-46.336 20.745-46.336 46.336h0zM512 157.312c-134.912 0-244.16 112.768-244.16 252.032 0 119.488 80.512 219.52 188.608 245.504-4.945-11.793-7.818-25.498-7.818-39.875 0-28.287 11.122-53.977 29.232-72.934l-0.038 0.041c17.747-18.568 42.71-30.11 70.368-30.11s52.621 11.543 70.333 30.074l0.035 0.037c12.992 13.632 35.84 32.256 60.224 51.328 47.786-46.337 77.443-111.131 77.443-182.853 0-0.426-0.001-0.851-0.003-1.276l0 0.066c-0.064-139.264-109.312-252.032-244.224-252.032zM773.312 752.256c-20.608-21.312-198.4-150.464-204.736-157.184-4.981-5.181-11.971-8.4-19.712-8.4s-14.731 3.219-19.703 8.391l-0.009 0.009c-5.055 5.277-8.168 12.451-8.168 20.352s3.112 15.075 8.178 20.363l-0.010-0.011c5.952 5.952 130.56 188.928 152.256 211.328 11.608 12.085 27.903 19.594 45.952 19.594s34.344-7.509 45.932-19.573l0.020-0.021c11.764-12.304 19.005-29.018 19.005-47.424s-7.241-35.12-19.030-47.45l0.024 0.026zM512 434.752c-24.96 0-30.912-5.12-30.912-30.912 0-25.728 5.952-30.848 30.912-30.848s30.912 5.12 30.912 30.912c0 25.728-5.952 30.848-30.912 30.848z"],"isMulticolor":false,"isMulticolor2":false,"tags":["storage"],"defaultCode":59023,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":12,"order":20,"ligatures":"","prevSize":32,"code":59023,"name":"storage"},"setIdx":1,"setId":5,"iconIdx":11},{"icon":{"paths":["M834.688 171.385c-39.44 0-71.709-32.269-71.709-71.709v-35.853h-537.813c-39.439 0-71.708 32.268-71.708 71.708v752.938c0 39.44 32.269 71.709 71.708 71.709h573.667c39.44 0 71.708-32.269 71.708-71.709v-717.084h-35.853zM393.368 423.874h-80.186v260.502h-32.033v-260.502h-79.783v-34.854h192.002v34.854zM583.758 684.376l-61.131-105.316c-3.414-5.88-6.861-12.481-10.338-19.808-5.504 10.773-9.448 18.148-11.834 22.114l-60.749 103.010h-38.48l93.843-153.998-82.763-141.358h38.1l44.27 75.652c9.087 15.529 15.503 27.675 19.236 36.445 5.195-10.524 11.611-21.737 19.213-33.646l48.978-78.451h34.494l-84.872 138.961 91.52 156.395h-39.487zM825.724 423.874h-80.186v260.502h-32.033v-260.502h-79.783v-34.854h192.002v34.854z"],"isMulticolor":false,"isMulticolor2":false,"tags":["txt"],"defaultCode":58936,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":13,"order":21,"ligatures":"","prevSize":32,"code":58936,"name":"txt"},"setIdx":1,"setId":5,"iconIdx":12},{"icon":{"paths":["M896 128s-129.9 49.1-203.6 26.5c-118.4-36.3-180.5-90.5-180.5-90.5s-62.1 54.2-180.4 90.5c-73.7 22.6-203.5-26.5-203.5-26.5s-17.6 270.9 9.5 474.5c27.1 203.6 374.4 357.5 374.4 357.5s347.6-153.9 374.6-357.5c27.1-203.6 9.5-474.5 9.5-474.5zM486 667l-64.5-64.5 0.3-0.3-71.3-71.2 64.2-64.2 71.3 71.2 207-207 64.5 64.5-271.5 271.5z"],"isMulticolor":false,"isMulticolor2":false,"tags":["quality"],"defaultCode":58919,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":14,"order":34,"ligatures":"","prevSize":32,"code":58919,"name":"quality"},"setIdx":1,"setId":5,"iconIdx":13},{"icon":{"paths":["M979.392 213.312c-7.552-114.88-209.856-203.904-467.392-203.904s-459.84 89.024-467.392 203.904h-0.704v590.72c0 118.016 205.568 210.56 468.096 210.56s468.096-92.544 468.096-210.56v-590.72h-0.704zM512 80.256c234.688 0 393.408 73.216 399.424 148.672-5.952 75.52-164.736 148.672-399.424 148.672s-393.408-73.152-399.424-148.608c5.952-75.52 164.736-148.672 399.424-148.672zM512 943.616c-239.232 0-400-76.032-400-153.024v-152.512c81.408 60.416 228.288 100.096 400 100.096s318.592-39.68 400-100.096v152.512c0 76.992-160.768 153.024-400 153.024zM912 502.912v11.264c0 76.992-160.768 153.024-400 153.024s-400-75.968-400-152.96v-165.76c81.408 60.352 228.288 100.096 400 100.096s318.592-39.68 400-100.096v154.432z"],"isMulticolor":false,"isMulticolor2":false,"tags":["hivedb-open"],"defaultCode":59059,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":15,"order":33,"ligatures":"","prevSize":32,"code":59059,"name":"hivedb-open"},"setIdx":1,"setId":5,"iconIdx":14},{"icon":{"paths":["M1023.457 780.8v-551.009c0-22.764-1.633-44.006-21.043-60.017-17.5-14.444-38.42-12.744-59.397-12.744h-449.357l22.103 12.682-72.34-101.125c-16.722-23.383-67.348-13.548-92.058-13.548h-271.334c-33.019 0-33.019 51.2 0 51.2h340.669l-22.103-12.682 54.912 76.764c12.416 17.357 22.093 37.903 45.85 37.903h465.981c10.286 0 6.917 16.748 6.917 23.475v549.1c0 33.014 51.2 33.014 51.2 0zM948.055 852.48h-875.259c-14.177 0-7.040-65.459-7.040-74.568v-608.794c0-6.932-5.868-61.368 5.12-62.479 32.527-3.282 32.845-54.513 0-51.2-63.35 6.39-56.32 64.369-56.32 110.812v642.483c0 41.262 0.061 94.95 57.733 94.95h875.766c33.014-0.005 33.014-51.205 0-51.205zM63.58 394.24h908.79c33.014 0 33.014-51.2 0-51.2h-908.785c-33.019 0-33.019 51.2-0.005 51.2z"],"isMulticolor":false,"isMulticolor2":false,"tags":["folder-o"],"defaultCode":58929,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":16,"order":32,"ligatures":"","prevSize":32,"code":58929,"name":"folder-o"},"setIdx":1,"setId":5,"iconIdx":15},{"icon":{"paths":["M399.587 72.8c-90.4 16-106.8 49.4-106.8 111.2v81.4h213.8v27.2h-294c-62.2 0-116.6 37.4-133.6 108.4-19.6 81.4-20.4 132.2 0 217.2 15.2 63.2 51.4 108.4 113.6 108.4h73.4v-97.6c0-70.6 61-132.8 133.6-132.8h213.6c59.4 0 106.8-49 106.8-108.6v-203.8c0-58-48.8-101.4-106.8-111.2-71.6-11.8-149.4-11.2-213.6 0.2zM386.187 129.6c22 0 40.2 18.4 40.2 40.8s-18 40.6-40.2 40.6c-22.2 0-40.2-18.2-40.2-40.6 0.2-22.6 18-40.8 40.2-40.8zM756.587 292.4v95c0 73.6-62.4 135.6-133.6 135.6h-213.6c-58.4 0-106.8 50-106.8 108.6v203.6c0 58 50.4 92 106.8 108.6 67.6 19.8 132.6 23.4 213.6 0 53.8-15.6 106.8-47 106.8-108.6v-81.4h-213.4v-27.2h320.4c62.2 0 85.2-43.4 106.8-108.4 22.4-67 21.4-131.4 0-217.2-15.4-61.8-44.6-108.4-106.8-108.4h-80.2zM636.387 808c22.2 0 40.2 18.2 40.2 40.6 0 22.6-18 40.8-40.2 40.8-22 0-40.2-18.4-40.2-40.8 0.2-22.6 18.2-40.6 40.2-40.6z"],"isMulticolor":false,"isMulticolor2":false,"tags":["python"],"defaultCode":61974,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":17,"order":31,"ligatures":"","prevSize":32,"code":61974,"name":"python"},"setIdx":1,"setId":5,"iconIdx":16},{"icon":{"paths":["M354.401 0c-87.040 0-157.44 70.559-157.44 157.599v275.681h-118.241c-21.658 0-39.363 17.7-39.363 39.363v236.319c0 21.658 17.7 39.357 39.363 39.357h118.241v118.083c0 87.040 70.4 157.599 157.44 157.599h472.637c87.040 0 157.599-70.559 157.599-157.599v-551.368c0-41.748-38.989-81.93-107.52-149.279l-58.24-58.235c-67.338-68.531-107.484-107.52-149.279-107.52h-315.192zM354.401 78.72h287.201c28.355 7.091 27.996 42.138 27.996 76.8v120.161c0 21.658 17.7 39.357 39.363 39.357h118.077c39.388 0 78.879-0.026 78.879 39.363v512c0 43.32-35.553 78.879-78.879 78.879h-472.632c-43.325 0-78.72-35.558-78.72-78.879v-118.083h393.917c21.663 0 39.363-17.695 39.363-39.357v-236.319c0-21.663-17.7-39.363-39.363-39.363h-393.923v-275.681c0-43.32 35.395-78.879 78.72-78.879zM312.801 485.919h20.157l65.28 176.64h-23.040l-19.2-54.717h-65.28l-19.2 54.717h-23.040l64.323-176.64zM204.319 486.881h22.083v172.8c0 16-4.163 28.16-12.483 36.48-8.32 8.96-20.157 13.44-35.517 13.44-7.040 0-13.123-0.963-18.243-2.883v-19.2c6.4 1.28 12.8 1.92 19.2 1.92 8.32 0 14.403-2.237 18.243-6.717 4.48-4.48 6.717-11.203 6.717-20.163v-175.677zM422.241 486.881h42.24c21.12 0 36.797 4.157 47.037 12.477s15.36 20.803 15.36 37.443c0 23.040-11.52 38.717-34.56 47.037l48 78.72h-25.917l-42.24-72.96h-27.848v72.96h-22.067v-175.677zM444.319 506.081v65.28h20.163c13.44 0 23.357-2.883 29.757-8.643s9.6-14.080 9.6-24.96-3.2-18.877-9.6-23.997c-5.76-5.12-16-7.68-30.72-7.68h-19.2zM323.359 509.921c-1.92 8.96-4.797 18.237-8.637 27.837l-17.28 50.883h51.84l-18.243-50.883c-3.84-10.88-6.4-20.157-7.68-27.837z"],"isMulticolor":false,"isMulticolor2":false,"tags":["jar"],"defaultCode":58904,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":18,"order":30,"ligatures":"","prevSize":32,"code":58904,"name":"jar"},"setIdx":1,"setId":5,"iconIdx":17},{"icon":{"paths":["M1145.56 245.477h-1145.56v-163.651c0-45.191 36.635-81.826 81.826-81.826v-0h286.39l163.651 163.651h531.867c45.191 0 81.826 36.635 81.826 81.826v-0zM81.826 327.303h981.909q81.826 0 81.826 81.826v490.954q0 81.826-81.826 81.826h-981.909q-81.826 0-81.826-81.826v-490.954q0-81.826 81.826-81.826z"],"width":1194,"isMulticolor":false,"isMulticolor2":false,"tags":["folder"],"defaultCode":58914,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":19,"order":29,"ligatures":"","prevSize":32,"code":58914,"name":"folder"},"setIdx":1,"setId":5,"iconIdx":18},{"icon":{"paths":["M883.197 1024h-639.997c-77.703-0.145-140.655-63.097-140.8-140.786l-0-0.014v-742.397c0.145-77.703 63.097-140.655 140.786-140.8l0.014-0h516.35c0.218-0.002 0.475-0.003 0.732-0.003 24.356 0 46.43 9.774 62.512 25.614l-0.011-0.011 189.695 189.695c7.114 6.975 11.523 16.686 11.523 27.426 0 0.168-0.001 0.336-0.003 0.503l0-0.025v639.998c-0.145 77.703-63.097 140.655-140.786 140.8l-0.014 0zM243.2 76.803c-35.346 0-64 28.654-64 64v742.397c0 35.346 28.654 64 64 64h639.998c35.346 0 64-28.654 64-64v-624.126l-179.2-179.2c-2.223-1.909-5.135-3.072-8.319-3.072-0.045 0-0.090 0-0.135 0.001l0.007-0zM292.095 512.002c0-73.728 44.8-118.016 102.4-118.016 25.463 0.095 48.334 11.108 64.191 28.599l0.065 0.073-19.456 22.784c-10.918-12.262-26.677-20.010-44.25-20.224l-0.038-0c-38.912 0-66.048 32.512-66.048 85.76s25.6 86.527 65.024 86.527c0.099 0.001 0.216 0.001 0.333 0.001 20.444 0 38.71-9.325 50.777-23.953l0.090-0.112 18.944 22.528c-16.461 19.924-41.173 32.526-68.829 32.526-0.552 0-1.104-0.005-1.654-0.015l0.083 0.001c-58.112 0.768-101.632-41.728-101.632-116.48zM478.207 595.969l20.992-25.6c15.444 15.431 36.626 25.123 60.070 25.599l0.090 0.001c27.391 0 42.751-12.8 42.751-32s-15.36-27.136-36.352-36.352l-31.232-13.567c-25.947-8.048-44.712-31.229-46.074-58.985l-0.006-0.151c3.391-34.17 31.986-60.638 66.763-60.638 2.816 0 5.591 0.174 8.315 0.51l-0.326-0.033c26.739 0.115 50.9 11.056 68.345 28.664l0.007 0.008-18.688 22.784c-12.859-12.413-30.352-20.1-49.64-20.224l-0.024-0c-23.040 0-38.144 11.008-38.144 29.184s18.176 25.6 36.608 34.048l30.976 13.312c26.53 7.898 45.548 32.061 45.568 60.669l0 0.002c0 36.352-29.952 65.792-79.616 65.792-0.208 0.001-0.454 0.002-0.7 0.002-31.122 0-59.296-12.622-79.682-33.025l-0.001-0.001zM650.75 399.106h37.888l33.536 116.992c7.68 25.6 12.544 47.871 20.48 73.983h1.536c7.68-25.6 13.312-48.128 20.48-73.983l33.536-116.992h36.351l-70.655 226.047h-42.24z"],"isMulticolor":false,"isMulticolor2":false,"tags":["csv"],"defaultCode":59562,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":20,"order":28,"ligatures":"","prevSize":32,"code":59562,"name":"csv"},"setIdx":1,"setId":5,"iconIdx":19},{"icon":{"paths":["M883.197 1024h-639.997c-77.703-0.145-140.655-63.097-140.8-140.786l-0-0.014v-742.397c0.145-77.703 63.097-140.655 140.786-140.8l0.014-0h516.35c0.218-0.002 0.475-0.003 0.732-0.003 24.356 0 46.43 9.774 62.512 25.614l-0.011-0.011 189.695 189.695c7.114 6.975 11.523 16.686 11.523 27.426 0 0.168-0.001 0.336-0.003 0.503l0-0.025v639.998c-0.145 77.703-63.097 140.655-140.786 140.8l-0.014 0zM243.2 76.803c-35.346 0-64 28.654-64 64v742.397c0 35.346 28.654 64 64 64h639.998c35.346 0 64-28.654 64-64v-624.126l-179.2-179.2c-2.223-1.909-5.135-3.072-8.319-3.072-0.045 0-0.090 0-0.135 0.001l0.007-0zM370.687 508.418l-60.928-109.312h39.68l27.392 52.736c5.632 10.496 10.496 20.48 17.152 34.048h1.536c5.888-13.568 10.24-23.552 15.36-34.048l25.6-52.736h37.376l-60.672 111.104 65.024 114.943h-38.4l-30.208-56.32-18.688-36.864c-6.4 13.824-12.032 25.6-17.408 36.864l-28.928 56.064h-38.656zM513.79 399.106h35.584v195.84h95.488v30.207h-131.072zM660.734 595.969l20.992-25.6c15.444 15.431 36.626 25.123 60.070 25.599l0.090 0.001c27.392 0 42.752-12.8 42.752-32s-15.36-27.136-36.352-36.352l-31.488-11.263c-25.947-8.048-44.712-31.229-46.074-58.985l-0.006-0.151c2.41-35.072 31.451-62.608 66.923-62.608 2.578 0 5.121 0.145 7.623 0.428l-0.307-0.028c26.739 0.115 50.9 11.056 68.345 28.664l0.007 0.008-18.688 22.784c-12.846-12.435-30.348-20.129-49.645-20.224l-0.019-0c-23.040 0-38.144 11.008-38.144 29.184s18.176 25.6 36.608 34.048l30.976 13.312c26.712 7.616 45.962 31.751 46.080 60.401l0 0.014c0 36.352-29.952 65.792-79.616 65.792-0.133 0.001-0.289 0.001-0.446 0.001-31.121 0-59.295-12.621-79.681-33.024l-0.001-0.001z"],"isMulticolor":false,"isMulticolor2":false,"tags":["xls"],"defaultCode":59588,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":21,"order":27,"ligatures":"","prevSize":32,"code":59588,"name":"xls"},"setIdx":1,"setId":5,"iconIdx":20},{"icon":{"paths":["M883.197 1024h-639.997c-77.703-0.145-140.655-63.097-140.8-140.786l-0-0.014v-742.397c0.145-77.703 63.097-140.655 140.786-140.8l0.014-0h516.35c0.218-0.002 0.475-0.003 0.732-0.003 24.356 0 46.43 9.774 62.512 25.614l-0.011-0.011 189.695 189.695c7.114 6.975 11.523 16.686 11.523 27.426 0 0.168-0.001 0.336-0.003 0.503l0-0.025v639.998c-0.145 77.703-63.097 140.655-140.786 140.8l-0.014 0zM243.2 76.803c-35.346 0-64 28.654-64 64v742.397c0 35.346 28.654 64 64 64h639.998c35.346 0 64-28.654 64-64v-624.126l-179.2-179.2c-2.223-1.909-5.135-3.072-8.319-3.072-0.045 0-0.090 0-0.135 0.001l0.007-0zM274.943 508.418l-60.928-109.312h39.68l27.904 52.736c5.632 10.496 10.496 20.48 17.152 34.048h1.536c5.888-13.568 10.24-23.552 15.36-34.048l25.6-52.736h37.376l-60.673 111.104 66.048 114.943h-39.936l-29.696-56.064-18.688-36.864c-6.4 13.824-12.032 25.6-17.408 36.864l-30.464 56.064h-37.632zM418.047 399.106h35.584v195.84h95.487v30.207h-131.071zM564.99 595.969l20.992-25.6c15.444 15.431 36.626 25.123 60.070 25.599l0.090 0.001c27.392 0 42.752-12.8 42.752-32s-15.36-27.136-36.352-36.352l-31.232-13.567c-25.947-8.048-44.712-31.229-46.074-58.985l-0.006-0.151c2.41-35.072 31.451-62.608 66.923-62.608 2.578 0 5.121 0.145 7.623 0.428l-0.307-0.028c26.635 0.98 50.383 12.58 67.276 30.664l0.052 0.056-18.688 22.784c-12.859-12.413-30.352-20.1-49.64-20.224l-0.024-0c-23.040 0-38.144 11.008-38.144 29.184s18.176 25.6 36.608 34.048l30.976 13.312c27.176 7.341 46.881 31.674 47.104 60.645l0 0.025c0 36.352-29.952 65.792-79.616 65.792-0.208 0.001-0.454 0.002-0.701 0.002-31.122 0-59.296-12.622-79.682-33.025l-0.001-0.001zM807.678 508.418l-60.672-109.312h39.68l27.392 52.736c5.632 10.496 10.496 20.48 17.152 34.048h1.536c5.887-13.568 10.24-23.552 15.36-34.048l25.6-52.736h37.374l-60.67 111.104 65.024 114.943h-38.912l-29.696-56.064-18.687-36.864c-6.4 13.824-12.032 25.6-17.408 36.864l-28.928 56.064h-39.424z"],"isMulticolor":false,"isMulticolor2":false,"tags":["xlsx"],"defaultCode":59589,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":22,"order":26,"ligatures":"","prevSize":32,"code":59589,"name":"xlsx"},"setIdx":1,"setId":5,"iconIdx":21},{"icon":{"paths":["M512 384c-229.8 0-416-57.3-416-128v256c0 70.7 186.2 128 416 128s416-57.3 416-128v-256c0 70.7-186.2 128-416 128zM512 704c-229.8 0-416-57.3-416-128v256c0 70.7 186.2 128 416 128s416-57.3 416-128v-256c0 70.7-186.2 128-416 128zM512 320c229.8 0 416-57.3 416-128s-186.2-128-416-128-416 57.3-416 128 186.2 128 416 128z"],"isMulticolor":false,"isMulticolor2":false,"tags":["hivedb"],"defaultCode":59049,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":23,"order":25,"ligatures":"","prevSize":32,"code":59049,"name":"hivedb"},"setIdx":1,"setId":5,"iconIdx":22},{"icon":{"paths":["M576 272v-272h-400c-26.6 0-48 21.4-48 48v928c0 26.6 21.4 48 48 48h672c26.6 0 48-21.4 48-48v-656h-272c-26.4 0-48-21.6-48-48zM896 243.8v12.2h-256v-256h12.2c12.8 0 25 5 34 14l195.8 196c9 9 14 21.2 14 33.8z"],"isMulticolor":false,"isMulticolor2":false,"tags":["file"],"defaultCode":58880,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":24,"order":24,"ligatures":"","prevSize":32,"code":58880,"name":"file"},"setIdx":1,"setId":5,"iconIdx":23},{"icon":{"paths":["M867.656 195.962l-167.767-167.767c-17.996-17.996-42.392-28.194-67.787-28.194h-408.12c-52.99 0.2-95.981 43.192-95.981 96.181v831.838c0 52.99 42.992 95.981 95.981 95.981h575.888c52.99 0 95.981-42.992 95.981-95.981v-664.070c0-25.395-10.198-49.99-28.194-67.987zM792.070 256.15h-152.17v-152.17l152.17 152.17zM223.981 928.019v-831.838h319.938v207.959c0 26.595 21.396 47.991 47.991 47.991h207.959v575.888h-575.888z"],"isMulticolor":false,"isMulticolor2":false,"tags":["file-o"],"defaultCode":59419,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":25,"order":23,"ligatures":"","prevSize":32,"code":59419,"name":"file-o"},"setIdx":1,"setId":5,"iconIdx":24},{"icon":{"paths":["M456.488 11.741c-17.718 0.542-35.227 8.963-49.859 25.221-7.738 8.768-14.569 18.646-20.159 29.278l-0.393 0.821c-14.799 27.806-17.718 58.989-23.179 89.338 11.325-84.678 0.758-26.893-9.591 30.968l-20.674 139.412c-1.126 6.67-4.002 9.463-10.172 11.381-73.663 23.179-147.451 46.274-220.823 70.203-17.709 5.563-33.090 12.725-47.297 21.593l0.856-0.498c-43.439 27.723-49.859 73.080-17.301 112.975 14.424 17.592 32.934 29.807 53.528 39.020 45.857 20.427 91.798 40.688 137.571 61.282 0.792 0.417 1.501 0.542 2.209 0.792h-0.167l54.445 22.303c0 0.959 0.083 1.876 0 3.043-8.338 84.627-16.258 169.379-24.179 254.090-1.668 17.467-0.5 34.643 5.586 51.277 13.215 36.269 44.398 47.858 78.416 29.223 15.508-8.504 27.764-20.719 39.062-34.101 57.196-67.66 114.476-135.237 171.464-203.105 4.627-5.545 8.546-6.253 15.133-4.085 14.549 4.878 29.057 9.63 43.564 14.466h0.042c0.834 0.5 1.876 0.834 3.043 1.251l5.92 1.751c52.027 17.342 111.725 32.308 163.793 49.609 16.675 5.503 25.93 6.837 43.564 4.085 12.965-2.001 24.679-2.585 34.518-9.838 25.096-13.757 41.48-39.896 41.188-70.12-0.208-17.801-6.42-33.893-14.591-49.442-34.184-64.867-68.244-129.776-102.595-194.517-3.252-6.17-3.168-10.13 1.542-15.633 57.571-67.327 114.893-134.903 172.298-202.397 11.339-13.34 21.386-27.431 27.222-44.106 12.84-36.602-3.752-65.451-41.688-72.579-5.842-1.043-12.568-1.639-19.432-1.639-11.418 0-22.451 1.649-32.873 4.723l0.821-0.208c-82.293 21.72-164.544 43.397-246.711 65.451-7.295 1.959-10.839 0.625-14.799-6.003-70.103-121.297-103.497-175.974-137.235-230.413l36.308 62.91c-8.075-13.393-17.060-24.991-27.223-35.437l0.043 0.044c-15.466-15.425-33.434-22.887-51.151-22.345l-0.042-0.042zM463.7 166.154c3.292 2.267 5.951 5.219 7.814 8.666l0.065 0.131c36.269 60.281 72.621 120.479 108.765 180.844 2.918 4.919 5.628 6.67 11.548 5.044 63.575-17.092 127.233-33.893 190.891-50.735 15.3-4.085 30.641-8.046 45.982-12.048-0.605 3.254-2.082 6.092-4.178 8.348l0.010-0.010c-52.486 61.699-104.888 123.481-157.54 185.013-4.586 5.419-5.003 9.171-1.668 15.425 39.062 75.164 73.705 137.946 104.137 198.436l-154.080-42.48c-22.428-7.504-44.898-14.924-67.327-22.47-6.795-2.293-10.422-1.251-15.008 4.169-52.027 62.032-104.387 123.814-156.706 185.68-1.91 2.475-4.453 4.374-7.389 5.465l-0.115 0.037c1.459-15.758 2.835-31.558 4.336-47.358 6.212-65.534 12.381-131.068 18.801-196.602-1.251-13.257 4.878-16.884-13.424-22.72-62.241-24.721-139.489-55.070-200.771-79.416 2.756-2.535 6.113-4.462 9.83-5.543l0.175-0.044c74.914-23.762 149.869-47.525 224.866-71.078 5.294-1.668 8.296-3.835 8.796-9.713 0.542-6.17 2.084-12.256 3.168-18.385 10.047-56.821 20.135-113.642 30.266-170.422 2.835-16.092 5.836-32.142 8.755-48.233z"],"isMulticolor":false,"isMulticolor2":false,"tags":["spark"],"defaultCode":60718,"grid":0,"attrs":[]},"attrs":[],"properties":{"id":26,"order":22,"ligatures":"","prevSize":32,"code":60718,"name":"spark"},"setIdx":1,"setId":5,"iconIdx":25}],"height":1024,"metadata":{"name":"iconfont"},"preferences":{"showGlyphs":true,"showQuickUse":true,"showQuickUse2":true,"showSVGs":true,"fontPref":{"prefix":"fi-","metadata":{"fontFamily":"iconfont","majorVersion":1,"minorVersion":0},"metrics":{"emSize":1024,"baseline":12.5,"whitespace":0},"embed":false,"showSelector":true,"showMetrics":true,"showMetadata":true,"showVersion":true},"imagePref":{"prefix":"icon-","png":true,"useClassSelector":true,"color":0,"bgColor":16777215,"classSelector":".icon"},"historySize":50,"showCodes":true,"gridSize":16}} \ No newline at end of file diff --git a/web/packages/dss/assets/projectIconFont/style.css b/web/packages/dss/assets/projectIconFont/style.css new file mode 100644 index 000000000..094aaed13 --- /dev/null +++ b/web/packages/dss/assets/projectIconFont/style.css @@ -0,0 +1,108 @@ +@font-face { + font-family: 'iconfont'; + src: url('fonts/iconfont.eot?av82gf'); + src: url('fonts/iconfont.eot?av82gf#iefix') format('embedded-opentype'), + url('fonts/iconfont.ttf?av82gf') format('truetype'), + url('fonts/iconfont.woff?av82gf') format('woff'), + url('fonts/iconfont.svg?av82gf#iconfont') format('svg'); + font-weight: normal; + font-style: normal; + font-display: block; +} + +[class^="fi-"], [class*=" fi-"] { + /* use !important to prevent issues with browser extensions that change fonts */ + font-family: 'iconfont' !important; + speak: never; + font-style: normal; + font-weight: normal; + font-variant: normal; + text-transform: none; + line-height: 1; + + /* Better Font Rendering =========== */ + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.fi-scriptis:before { + content: "\e900"; +} +.fi-guanlishitu:before { + content: "\e608"; +} +.fi-JDBC:before { + content: "\e75e"; +} +.fi-scala:before { + content: "\e727"; +} +.fi-hive:before { + content: "\e601"; +} +.fi-spark-python:before { + content: "\e603"; +} +.fi-mail:before { + content: "\e60d"; +} +.fi-python-o:before { + content: "\e624"; +} +.fi-data:before { + content: "\e6ab"; +} +.fi-log:before { + content: "\e650"; +} +.fi-table:before { + content: "\e955"; +} +.fi-field:before { + content: "\e612"; +} +.fi-storage:before { + content: "\e68f"; +} +.fi-txt:before { + content: "\e638"; +} +.fi-quality:before { + content: "\e627"; +} +.fi-hivedb-open:before { + content: "\e6b3"; +} +.fi-folder-o:before { + content: "\e631"; +} +.fi-python:before { + content: "\f216"; +} +.fi-jar:before { + content: "\e618"; +} +.fi-folder:before { + content: "\e622"; +} +.fi-csv:before { + content: "\e8aa"; +} +.fi-xls:before { + content: "\e8c4"; +} +.fi-xlsx:before { + content: "\e8c5"; +} +.fi-hivedb:before { + content: "\e6a9"; +} +.fi-file:before { + content: "\e600"; +} +.fi-file-o:before { + content: "\e81b"; +} +.fi-spark:before { + content: "\ed2e"; +} diff --git a/web/packages/dss/assets/styles/app.scss b/web/packages/dss/assets/styles/app.scss new file mode 100644 index 000000000..2c1567572 --- /dev/null +++ b/web/packages/dss/assets/styles/app.scss @@ -0,0 +1,126 @@ +@import '@dataspherestudio/shared/common/style/variables.scss'; +@import '@dataspherestudio/shared/common/style/overwrite.scss'; +@import '../projectIconFont/style.css'; +@import 'normalize.scss'; + +* { + box-sizing: border-box; + -webkit-tap-highlight-color: rgba(0, 0, 0, 0); +} + +*:after, +*:before { + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC", "Hiragino Sans GB", "Microsoft YaHei", "Helvetica Neue", Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; + font-variant-numeric: tabular-nums; + font-size: $font-size-small; + @include font-color($text-color, $dark-text-color); + line-height: $line-height-base; + background-color: $workspace-background !important; +} + +.layout { + position: $absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; + @include bg-color($body-background, $dark-base-color); + overflow: hidden; +} + +.layout-content { + height: 100%; +} + +.layout-body { + width: 100%; + height: 100%; + box-sizing: border-box; + // padding: 54px 0 0; + position: $relative; +} + +.layout-top { + padding: 54px 0 0; +} + +.ivu-message { + z-index: 2000 !important; +} + +/* 设置滚动条的样式 */ +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +/* 滚动槽 */ +::-webkit-scrollbar-track { + box-shadow: inset 0 0 2px $shadow-color; + -webkit-box-shadow: inset 0 0 2px $shadow-color; + border-radius: $border-radius-large; + -webkit-border-radius: $border-radius-large; + -moz-border-radius: $border-radius-large; + -ms-border-radius: $border-radius-large; + -o-border-radius: $border-radius-large; +} + +/* 滚动条滑块 */ +::-webkit-scrollbar-thumb { + box-shadow: inset 0 0 2px $shadow-color; + border-radius: $border-radius-large; + background: $scrollbar-color; + -webkit-box-shadow: inset 0 0 2px $shadow-color; + -webkit-border-radius: $border-radius-large; + -moz-border-radius: $border-radius-large; + -ms-border-radius: $border-radius-large; + -o-border-radius: $border-radius-large; +} + +::-webkit-scrollbar-thumb:window-inactive { + background: $scrollbar-color; +} + +.we-editor .monaco-editor .margin { + background-color: #f2f3f6; +} + +.result-table-content { + .__view::-webkit-scrollbar { + width: 0px; + height: 0px; + } + .table-bottom::-webkit-scrollbar { + width: 16px; + height: 16px; + } +} + +.ivu-modal-header { + // background-color: #FAFAFA !important; + @include bg-color(#FAFAFA, $dark-base-color); +} + +.notransition * { + -webkit-transition: none !important; + -moz-transition: none !important; + -o-transition: none !important; + -ms-transition: none !important; + transition: none !important; +} + +.overlay { + position: fixed; + padding: 0; + margin: 0; + top: 0; + left: 0; + background: rgba(0, 0, 0, 0.7); + width: 100%; + height: 100%; + z-index: 1001; +} diff --git a/web/packages/dss/assets/styles/login.scss b/web/packages/dss/assets/styles/login.scss new file mode 100644 index 000000000..bfd1278be --- /dev/null +++ b/web/packages/dss/assets/styles/login.scss @@ -0,0 +1,90 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +@import '@dataspherestudio/shared/common/style/variables.scss'; +.login { + position: $absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; + background-color: $body-background; + display: flex; + align-items: center; + justify-content: flex-end; + .login-bg { + position: $absolute; + width: 100%; + height: 100%; + top: 0; + left: 0; + background: url('../images/loginbgc.svg') no-repeat; + background-position: 55% 40%; + background-color: #001C40; + } + .login-main { + width: 450px; + height: 427px; + background-color: $body-background; + margin-right: 9.5%; + padding: 50px; + border-radius: 6px; + box-shadow: 2px 2px 40px 0px rgba(0, 0, 0, 0.9); + z-index: $zindex-spin; + .login-title { + font-size: 20px; + font-weight: 400px; + display: block; + text-align: center; + color: #044B93; + font-weight: 600; + } + .label { + font-size: 14px; + color: #333; + } + input { + border: none; + height: 50px; + border-radius: 4px; + padding-left: 15px; + color: $input-color; + border: 1px solid #DEE4EC; + background-color: $body-background !important; + } + button { + height: 45px; + background-color: $btn-primary-bg; + box-shadow: 0px 5px 10px 0px $shadow-color; + border-color: $btn-primary-bg; + } + } + .remember-user-name { + margin: 5px 0 0 10px; + color: #333; + } + .ivu-form-item { + margin-bottom: 20px; + } +} +.radioGroup { + padding-left: 20px; + display: block; + .ivu-radio-group-item { + width: 100%; + } +} diff --git a/web/packages/dss/assets/styles/normalize.scss b/web/packages/dss/assets/styles/normalize.scss new file mode 100644 index 000000000..7798b64ef --- /dev/null +++ b/web/packages/dss/assets/styles/normalize.scss @@ -0,0 +1,483 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/*! normalize.css v5.0.0 | MIT License | github.com/necolas/normalize.css */ + +/** + * 1. Change the default font family in all browsers (opinionated). + * 2. Correct the line height in all browsers. + * 3. Prevent adjustments of font size after orientation changes in + * IE on Windows Phone and in iOS. + */ + +/* Document + ========================================================================== */ + +html { + font-family: sans-serif; /* 1 */ + line-height: 1.15; /* 2 */ + -ms-text-size-adjust: 100%; /* 3 */ + -webkit-text-size-adjust: 100%; /* 3 */ +} + +/* Sections + ========================================================================== */ + +/** + * Remove the margin in all browsers (opinionated). + */ + +body { + margin: 0; +} + +/** + * Add the correct display in IE 9-. + */ + +article, +aside, +footer, +header, +nav, +section { + display: block; +} + +/** + * Correct the font size and margin on `h1` elements within `section` and + * `article` contexts in Chrome, Firefox, and Safari. + */ + +h1 { + font-size: 2em; + margin: 0.67em 0; +} + +/* Grouping content + ========================================================================== */ + +/** + * Add the correct display in IE 9-. + * 1. Add the correct display in IE. + */ + +figcaption, +figure, +main { /* 1 */ + display: block; +} + +/** + * Add the correct margin in IE 8. + */ + +figure { + margin: 1em 40px; +} + +/** + * 1. Add the correct box sizing in Firefox. + * 2. Show the overflow in Edge and IE. + */ + +hr { + box-sizing: content-box; /* 1 */ + height: 0; /* 1 */ + overflow: visible; /* 2 */ +} + +/** + * 1. Correct the inheritance and scaling of font size in all browsers. + * 2. Correct the odd `em` font sizing in all browsers. + */ + +pre { + font-family: monospace, monospace; /* 1 */ + font-size: 1em; /* 2 */ +} + +/* Text-level semantics + ========================================================================== */ + +/** + * 1. Remove the gray background on active links in IE 10. + * 2. Remove gaps in links underline in iOS 8+ and Safari 8+. + */ + +a { + background-color: transparent; /* 1 */ + -webkit-text-decoration-skip: objects; /* 2 */ +} + +/** + * Remove the outline on focused links when they are also active or hovered + * in all browsers (opinionated). + */ + +a:active, +a:hover { + outline-width: 0; +} + +/** + * 1. Remove the bottom border in Firefox 39-. + * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari. + */ + +abbr[title] { + border-bottom: none; /* 1 */ + text-decoration: underline; /* 2 */ + text-decoration: underline dotted; /* 2 */ +} + +/** + * Prevent the duplicate application of `bolder` by the next rule in Safari 6. + */ + +b, +strong { + font-weight: inherit; +} + +/** + * Add the correct font weight in Chrome, Edge, and Safari. + */ + +b, +strong { + font-weight: bolder; +} + +/** + * 1. Correct the inheritance and scaling of font size in all browsers. + * 2. Correct the odd `em` font sizing in all browsers. + */ + +code, +kbd, +samp { + font-family: monospace, monospace; /* 1 */ + font-size: 1em; /* 2 */ +} + +/** + * Add the correct font style in Android 4.3-. + */ + +dfn { + font-style: italic; +} + +/** + * Add the correct background and color in IE 9-. + */ + +mark { + background-color: #ff0; + color: #000; +} + +/** + * Add the correct font size in all browsers. + */ + +small { + font-size: 80%; +} + +/** + * Prevent `sub` and `sup` elements from affecting the line height in + * all browsers. + */ + +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} + +sub { + bottom: -0.25em; +} + +sup { + top: -0.5em; +} + +/* Embedded content + ========================================================================== */ + +/** + * Add the correct display in IE 9-. + */ + +audio, +video { + display: inline-block; +} + +/** + * Add the correct display in iOS 4-7. + */ + +audio:not([controls]) { + display: none; + height: 0; +} + +/** + * Remove the border on images inside links in IE 10-. + */ + +img { + border-style: none; +} + +/** + * Hide the overflow in IE. + */ + +svg:not(:root) { + overflow: hidden; +} + +/* Forms + ========================================================================== */ + +/** + * 1. Change the font styles in all browsers (opinionated). + * 2. Remove the margin in Firefox and Safari. + */ + +button, +input, +optgroup, +select, +textarea { + font-family: sans-serif; /* 1 */ + font-size: 100%; /* 1 */ + line-height: 1.15; /* 1 */ + margin: 0; /* 2 */ +} + +/** + * Show the overflow in IE. + * 1. Show the overflow in Edge. + */ + +button, +input { /* 1 */ + overflow: visible; +} + +/** + * Remove the inheritance of text transform in Edge, Firefox, and IE. + * 1. Remove the inheritance of text transform in Firefox. + */ + +button, +select { /* 1 */ + text-transform: none; +} + +/** + * 1. Prevent a WebKit bug where (2) destroys native `audio` and `video` + * controls in Android 4. + * 2. Correct the inability to style clickable types in iOS and Safari. + */ + +button, +html [type="button"], /* 1 */ +[type="reset"], +[type="submit"] { + -webkit-appearance: button; /* 2 */ +} + +/** + * Remove the inner border and padding in Firefox. + */ + +button::-moz-focus-inner, +[type="button"]::-moz-focus-inner, +[type="reset"]::-moz-focus-inner, +[type="submit"]::-moz-focus-inner { + border-style: none; + padding: 0; +} + +/** + * Restore the focus styles unset by the previous rule. + */ + +button:-moz-focusring, +[type="button"]:-moz-focusring, +[type="reset"]:-moz-focusring, +[type="submit"]:-moz-focusring { + outline: 1px dotted ButtonText; +} + +/** + * Change the border, margin, and padding in all browsers (opinionated). + */ + +fieldset { + border: 1px solid #c0c0c0; + margin: 0 2px; + padding: 0.35em 0.625em 0.75em; +} + +/** + * 1. Correct the text wrapping in Edge and IE. + * 2. Correct the color inheritance from `fieldset` elements in IE. + * 3. Remove the padding so developers are not caught out when they zero out + * `fieldset` elements in all browsers. + */ + +legend { + box-sizing: border-box; /* 1 */ + color: inherit; /* 2 */ + display: table; /* 1 */ + max-width: 100%; /* 1 */ + padding: 0; /* 3 */ + white-space: normal; /* 1 */ +} + +/** + * 1. Add the correct display in IE 9-. + * 2. Add the correct vertical alignment in Chrome, Firefox, and Opera. + */ + +progress { + display: inline-block; /* 1 */ + vertical-align: baseline; /* 2 */ +} + +/** + * Remove the default vertical scrollbar in IE. + */ + +textarea { + overflow: auto; +} + +/** + * 1. Add the correct box sizing in IE 10-. + * 2. Remove the padding in IE 10-. + */ + +[type="checkbox"], +[type="radio"] { + box-sizing: border-box; /* 1 */ + padding: 0; /* 2 */ +} + +/** + * Correct the cursor style of increment and decrement buttons in Chrome. + */ + +[type="number"]::-webkit-inner-spin-button, +[type="number"]::-webkit-outer-spin-button { + height: auto; +} + +/** + * 1. Correct the odd appearance in Chrome and Safari. + * 2. Correct the outline style in Safari. + */ + +[type="search"] { + -webkit-appearance: textfield; /* 1 */ + outline-offset: -2px; /* 2 */ +} + +/** + * Remove the inner padding and cancel buttons in Chrome and Safari on macOS. + */ + +[type="search"]::-webkit-search-cancel-button, +[type="search"]::-webkit-search-decoration { + -webkit-appearance: none; +} + +/** + * 1. Correct the inability to style clickable types in iOS and Safari. + * 2. Change font properties to `inherit` in Safari. + */ + +::-webkit-file-upload-button { + -webkit-appearance: button; /* 1 */ + font: inherit; /* 2 */ +} + +/* Interactive + ========================================================================== */ + +/* + * Add the correct display in IE 9-. + * 1. Add the correct display in Edge, IE, and Firefox. + */ + +details, /* 1 */ +menu { + display: block; +} + +/* + * Add the correct display in all browsers. + */ + +summary { + display: list-item; +} + +/* Scripting + ========================================================================== */ + +/** + * Add the correct display in IE 9-. + */ + +canvas { + display: inline-block; +} + +/** + * Add the correct display in IE. + */ + +template { + display: none; +} + +/* Hidden + ========================================================================== */ + +/** + * Add the correct display in IE 10-. + */ + +[hidden] { + display: none; +} + +ul, +ol { + list-style: none; +} diff --git a/web/packages/dss/babel.config.js b/web/packages/dss/babel.config.js new file mode 100644 index 000000000..ef8e58391 --- /dev/null +++ b/web/packages/dss/babel.config.js @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +module.exports = { + presets: [ + '@vue/app' + ] +} diff --git a/web/packages/dss/mock.js b/web/packages/dss/mock.js new file mode 100644 index 000000000..82953f9b4 --- /dev/null +++ b/web/packages/dss/mock.js @@ -0,0 +1,19 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +module.exports = function () { +} diff --git a/web/packages/dss/module/footer/guide.vue b/web/packages/dss/module/footer/guide.vue new file mode 100644 index 000000000..b12e083ab --- /dev/null +++ b/web/packages/dss/module/footer/guide.vue @@ -0,0 +1,642 @@ + + + diff --git a/web/packages/dss/module/footer/index.js b/web/packages/dss/module/footer/index.js new file mode 100644 index 000000000..7a9143eab --- /dev/null +++ b/web/packages/dss/module/footer/index.js @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export default { + name: 'Footer', + events: ['Footer:updateRunningJob', 'Footer:getRunningJob'], + component: () => import('./index.vue'), +}; diff --git a/web/packages/dss/module/footer/index.scss b/web/packages/dss/module/footer/index.scss new file mode 100644 index 000000000..92cfa8b4e --- /dev/null +++ b/web/packages/dss/module/footer/index.scss @@ -0,0 +1,96 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + @import '@dataspherestudio/shared/common/style/variables.scss'; + .layout-footer { + position: $fixed; + right: 0; + bottom: 0; + top: 0; + left: 0; + pointer-events: none; + width: 100%; + background: transparent; + overflow: hidden; + z-index: 1000; + text-align: center; + cursor: move; + .tool-btns { + pointer-events: initial; + position: absolute; + width: 40px; + right: 40px; + bottom: 20px; + height: 150px; + } + &:hover { + .min_arrow { + visibility: visible; + } + } + .min_arrow, .show_arrow { + font-size: 16px; + padding: 5px; + cursor: pointer; + &:hover { + @include font-color($primary-color, $dark-primary-color); + } + } + .min_arrow { + visibility: hidden; + } + } + + .footer-btn { + display: flex; + align-items: center; + justify-content: center; + border: $border-width-base $border-style-base $border-color-base; + border-radius: 8px; + box-shadow: 0px 1px 4px rgba(0, 0, 0, .2); + height: 40px; + line-height: 40px; + margin: 20px 0; + cursor: pointer; + font-size: 16px; + } + .footer-doc { + @include font-color(#333, #fff); + @include bg-color(#fff, #373F50); + @include border-color($border-color-base, #373F50); + } + + .footer-channel { + border: $border-width-base $border-style-base $border-color-base; + border-radius: $border-radius-small; + box-shadow: $shadow-base; + background-image: linear-gradient(39deg, $background-color-select-hover 0%, $background-color-base 100%); + display: flex; + align-items: center; + justify-content: center; + opacity: 0.8; + cursor: move; + color: $text-color; + .footer-channel-job { + font-weight: 900; + margin-right: 0px; + } + .footer-channel-job-num { + font-size: $font-size-small; + color: $text-color; + } + } diff --git a/web/packages/dss/module/footer/index.vue b/web/packages/dss/module/footer/index.vue new file mode 100644 index 000000000..4ae44f4d6 --- /dev/null +++ b/web/packages/dss/module/footer/index.vue @@ -0,0 +1,142 @@ + + + diff --git a/web/packages/dss/module/footer/libraryDetail.vue b/web/packages/dss/module/footer/libraryDetail.vue new file mode 100644 index 000000000..89426df99 --- /dev/null +++ b/web/packages/dss/module/footer/libraryDetail.vue @@ -0,0 +1,102 @@ + + + diff --git a/web/packages/dss/module/footer/libraryHome.vue b/web/packages/dss/module/footer/libraryHome.vue new file mode 100644 index 000000000..41a78f52c --- /dev/null +++ b/web/packages/dss/module/footer/libraryHome.vue @@ -0,0 +1,159 @@ + + + diff --git a/web/packages/dss/module/footer/librarySearch.vue b/web/packages/dss/module/footer/librarySearch.vue new file mode 100644 index 000000000..2aad2b0f0 --- /dev/null +++ b/web/packages/dss/module/footer/librarySearch.vue @@ -0,0 +1,84 @@ + + + diff --git a/web/packages/dss/module/header/index.js b/web/packages/dss/module/header/index.js new file mode 100644 index 000000000..f94a4b898 --- /dev/null +++ b/web/packages/dss/module/header/index.js @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export default { + name: 'Header', + component: () => import('./index.vue'), + dispatchs: ['Footer:getRunningJob', 'dssIndexedDB:deleteDb'], +}; diff --git a/web/packages/dss/module/header/index.scss b/web/packages/dss/module/header/index.scss new file mode 100644 index 000000000..f09e9a8b6 --- /dev/null +++ b/web/packages/dss/module/header/index.scss @@ -0,0 +1,155 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +@import '@dataspherestudio/shared/common/style/variables.scss'; + +@import '@dataspherestudio/shared/common/style/headerCommon.scss'; + + +.proj-select { + margin-left: 25px; + vertical-align: middle; + color: #fff; + margin-top: -2px; + display: flex; +} +.project-dro { + vertical-align: top; + height: 100%; + margin-left: 80px; + .project { + font-size: 14px; + display: flex; + justify-content: center; + align-items: center; + flex-wrap: nowrap; + height: 54px; + &:hover { + color:#fff; + } + } + .proj-list { + height: 400px; + overflow-y: auto; + @include bg-color($background-color-header, $dark-menu-base-color); + padding: 0 15px; + .proj-name { + padding:7px 15px; + height: 30px; + line-height: 30px; + font-size: $font-size-large; + border-bottom: 1px solid rgba(255,255,255,0.08); + @include border-color(rgba(255,255,255,0.08), $dark-border-color-base); + padding-bottom: 35px; + color: $text-tille-light-color; + font-size: 16px; + &::before { + content: ''; + font-weight: bold; + padding-left: 12px; + border-left: 3px solid #2d8cf0; + color: rgba(255,255,255,0.85); + } + } + .name-bar { + display: flex; + justify-content: flex-start; + align-items: center; + flex-wrap: wrap; + padding: 0 $padding-25; + .proj-item { + // width: 200px; + padding: 0 10px; + margin-right: 15px; + font-size: $font-size-14; + color: $text-tille-light-color; + cursor: pointer; + &:hover { + color: $dark-selected-color + } + &.active { + color: $dark-active-color + } + } + } + } + /deep/ .ivu-select-dropdown { + padding: 0; + margin: 0; + } +} +.menu-item-dropdown { + ::v-deep .ivu-select-dropdown { + max-width: inherit; + padding: 0; + margin: 0; + } + .list { + @include bg-color($background-color-header, $dark-menu-base-color); + padding: 5px 5px 10px; + color: $text-tille-light-color; + cursor: pointer; + ::v-deep .ivu-dropdown-item { + color: $text-tille-light-color; + &:hover { + background: none; + } + } + &:hover { + color: $dark-selected-color + } + } +} +.workspace-icon { + margin: 0 10px 0 25px; + display: inline-block; +} + +.luban-menu-trigger { + display: block; + margin-left: -20px; + width: 54px; + height: 54px; + line-height: 54px; + // background: #061528; + @include bg-color(#061528, $dark-menu-base-color); + text-align: center; + cursor: pointer; + &:hover { + // background: #2E92F7; + @include bg-color($primary-color, $dark-primary-color); + } + img { + width: 24px; + height: 24px; + vertical-align: middle; + } + +} +.header-actived { + color: $body-background; +} + +.header-actived:before { + content: ''; + height: 2px; + width: 100%; + margin-left: -10px; + background-color: #2E92F7; + position: absolute; + top: 40px; +} diff --git a/web/packages/dss/module/header/index.vue b/web/packages/dss/module/header/index.vue new file mode 100644 index 000000000..1c1db1fc5 --- /dev/null +++ b/web/packages/dss/module/header/index.vue @@ -0,0 +1,614 @@ + + + diff --git a/web/packages/dss/module/header/productMenu.vue b/web/packages/dss/module/header/productMenu.vue new file mode 100644 index 000000000..d96f7e0ee --- /dev/null +++ b/web/packages/dss/module/header/productMenu.vue @@ -0,0 +1,498 @@ + + + + diff --git a/web/packages/dss/module/header/userMenu.vue b/web/packages/dss/module/header/userMenu.vue new file mode 100644 index 000000000..620b0d598 --- /dev/null +++ b/web/packages/dss/module/header/userMenu.vue @@ -0,0 +1,150 @@ + + + diff --git a/web/packages/dss/module/index.js b/web/packages/dss/module/index.js new file mode 100644 index 000000000..b9be57b2d --- /dev/null +++ b/web/packages/dss/module/index.js @@ -0,0 +1,19 @@ +import mixinDispatch from '@dataspherestudio/shared/common/service/moduleMixin'; + +const requireComponent = require.context( + // 其组件目录的相对路径 + './', + // 是否查询其子目录 + true, + /([a-z|A-Z])+\/index\.js$/ +); + +const requireComponentVue = require.context( + // 其组件目录的相对路径 + './', + // 是否查询其子目录 + true, + /([a-z|A-Z])+.vue$/ +); + +mixinDispatch(requireComponent, requireComponentVue) \ No newline at end of file diff --git a/web/packages/dss/module/indexedDB/index.js b/web/packages/dss/module/indexedDB/index.js new file mode 100644 index 000000000..7bf202977 --- /dev/null +++ b/web/packages/dss/module/indexedDB/index.js @@ -0,0 +1,10 @@ +import { db } from '@dataspherestudio/shared/common/service/db/index.js'; + +export default { + name: 'dssIndexedDB', + methods: { + deleteDb() { + db.db.delete(); + }, + } +} \ No newline at end of file diff --git a/web/packages/dss/module/newGuidance/index.js b/web/packages/dss/module/newGuidance/index.js new file mode 100644 index 000000000..4b44ddb31 --- /dev/null +++ b/web/packages/dss/module/newGuidance/index.js @@ -0,0 +1,4 @@ +export default { + name: 'newComerGuidance', + component: () => import('./index.vue'), +}; \ No newline at end of file diff --git a/web/packages/dss/module/newGuidance/index.scss b/web/packages/dss/module/newGuidance/index.scss new file mode 100644 index 000000000..2e37f034e --- /dev/null +++ b/web/packages/dss/module/newGuidance/index.scss @@ -0,0 +1,1002 @@ +// 首页、第九步 +#mask1, #mask9 { + position : fixed; + top : 50%; + left : 50%; + transform: translate(-50%, -50%); + width : 305px; + z-index : 1002; + font-size: 14px; + + .description { + + font-size : 14px; + color : #ffffff; + border : 2px dashed #ffffff; + padding : 16px; + position: relative; + + p { + text-align: justify; + span { + color: rgb(9, 140, 234); + font-weight: 550; + } + } + &::after{ + content: ""; + width: 10px; + height: 10px; + transform: rotate(45deg); + border-width: 0 2px 2px 0; + border-color: #fff; + border-style: dashed; + position : absolute; + bottom : -7px; + left : 110px; + background-color: #4c4c4c; + } + } + + #confirm1 { + padding-left: 138px; + display : flex; + padding-top: 36px; + + .next { + + color : #FFFFFF; + text-align : center; + line-height : 35px; + width : 100px; + height : 35px; + background : #2E92F7; + border-radius: 22px; + cursor : pointer; + } + + .luBanLogo { + width : 60px; + height : 57px; + margin-top : -15px; + margin-left: -149px; + } + } +} + +#mask1 { + .description{ + p{ + &:first-child{ + font-weight: bold; + } + } + } +} + + +// 第二步 +#mask2 { + position : fixed; + top : 50%; + left : 50%; + transform: translate(-50%, -50%); + width : 215px; + z-index : 1003; + + .description { + position: relative; + + font-size : 14px; + color : #ffffff; + border : 2px dashed #ffffff; + padding : 16px; + + p { + text-align: justify; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + } + + &::after{ + content: ""; + width: 10px; + height: 10px; + transform: rotate(45deg); + border-width: 0 2px 2px 0; + border-color: #fff; + border-style: dashed; + position : absolute; + bottom : -7px; + left : 164.5px; + background-color: #4c4c4c; + } + } + + #confirm2 { + display : flex; + justify-content: center; + align-items : center; + font-size : 14px; + margin-left : 12px; + padding-top: 24px; + + .skip { + + color : rgba(255, 255, 255, 0.65); + cursor : pointer; + } + + .next { + + color : #FFFFFF; + text-align : center; + width : 84px; + height : 35px; + line-height : 35px; + background : #2e92f7; + border-radius: 22px; + margin-left : 16px; + cursor : pointer; + } + + .luBanLogo { + width : 60px; + height : 57px; + margin-top : -5px; + margin-left: -8px; + } + } +} + +.headerSuspension { + position: fixed; + top : 0; + left : 0; + width : 100%; + z-index : 1003; + + .item1 { + .dashed { + display : inline-block; + height : 54px; + border : 2px dashed #ffffff; + margin-top: 0.3px; + } + + .block1 { + width: 53px; + } + + .block2 { + margin-left: 8px; + width : 258px; + } + + .block3 { + width : 110px; + margin-left: 110px; + } + + .block4 { + width : 80px; + margin-left: 35px; + } + + .block5 { + width : 54px; + position: absolute; + top : 0px; + right : 5px; + } + } + + .item2 { + .img1 { + margin-left: 25px; + } + + .img2 { + margin-left: 116px; + } + + .img3 { + margin-left: 245px; + } + + .img4 { + margin-left: 116px; + } + + .img5 { + position : absolute; + margin-left: 128px; + right : 30px; + } + } + + .item3 { + + font-size : 14px; + color : #ffffff; + text-align : justify; + min-width : 1050px; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + + div { + display : inline-block; + height : 60px; + vertical-align: middle; + } + + .desc1 { + width : 150px; + margin-left: 25px; + } + + .desc2 { + width : 130px; + margin-left: 40px; + } + + .desc3 { + width : 130px; + margin-left: 125px; + } + + .desc4 { + width : 130px; + margin-left: 40px; + } + + .desc5 { + width : 154px; + right : 30px; + position: absolute; + } + } +} + +.footerSuspension { + position: absolute; + bottom : 0; + right : 0; + z-index : 1003; + width: 383px; + height: 250px; + + .item1 { + + font-size : 14px; + color : #ffffff; + text-align : justify; + display : inline-block; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + + div { + height: 60px; + width : 180px; + } + + .desc2 { + width : 200px; + margin-top : 65px; + } + } + + .item2 { + width : 85px; + display : inline-block; + margin-right: 5px; + + .img1 { + margin-top: 45px; + } + + .img2 { + margin-top : 50px; + margin-bottom: 30px; + } + } + + .item3 { + position: relative; + bottom: -48px; + display : inline-block; + + .dashed { + width : 65px; + height: 64px; + border: 2px dashed #ffffff; + margin-right: 27px; + } + } +} + +// 第三步 +#mask3 { + position: fixed; + top : 0; + left : 0; + z-index : 1004; + + .selected { + height : 54px; + border : 2px dashed #ffffff; + margin-top: 0.3px; + width : 54px; + cursor: pointer; + &:hover{ + background-color: #ffffff8c; + } + } + + .arrow { + margin-left: 25px; + } + + .desc { + margin-left: 98px; + + font-size : 14px; + color : #ffffff; + border : 2px dashed #ffffff; + padding : 16px; + width : 200px; + position: relative; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + + &::after{ + content: ""; + width: 10px; + height: 10px; + transform: rotate(45deg); + border-width: 0 2px 2px 0; + border-color: #fff; + border-style: dashed; + position : absolute; + bottom: -7px; + left: 158px; + background-color: #4c4c4c; + } + } + + #confirm3 { + margin-left : 8px; + display : flex; + justify-content: center; + align-items : center; + text-align : center; + padding-top: 27px; + + .skip { + + font-size : 14px; + color : rgba(255, 255, 255, 0.65); + width : 46px; + padding-right: 16px; + cursor : pointer; + } + + .prev { + + font-size : 14px; + color : #ffffff; + border : 1px dashed #ffffff; + border-radius: 22px; + width : 84px; + height : 35px; + line-height : 35px; + margin-right : 16px; + cursor : pointer; + } + + .next { + + font-size : 14px; + color : #ffffff; + width : 84px; + height : 35px; + background : #2e92f7; + border-radius: 22px; + line-height : 35px; + cursor : pointer; + } + + .luBanLogo { + width : 60px; + height : 57px; + margin-top : -5px; + margin-left: -8px; + } + } +} + +// 第四步 +#mask4 { + display : flex; + position: fixed; + top : 54px; + left : 0; + z-index : 2300; + bottom : 0; + + .selected { + border : 2px dashed #ffffff; + margin-top: 0.3px; + width : 810px; + height : 100%; + } + + .right { + position : relative; + margin-top: 82px; + + .desc { + margin-left: 98px; + + font-size : 14px; + color : #ffffff; + border : 2px dashed #ffffff; + padding : 16px; + width : 200px; + position: relative; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + + &::after{ + content: ""; + width: 10px; + height: 10px; + transform: rotate(45deg); + border-width: 0 2px 2px 0; + border-color: #fff; + border-style: dashed; + position : absolute; + bottom : -7px; + left : 158px; + background-color: #4c4c4c; + } + } + + + #confirm4 { + margin-left : 8px; + display : flex; + justify-content: center; + align-items : center; + text-align : center; + padding-top: 27px; + + .skip { + + font-size : 14px; + color : rgba(255, 255, 255, 0.65); + width : 46px; + padding-right: 16px; + cursor : pointer; + } + + .prev { + + font-size : 14px; + color : #ffffff; + border : 1px dashed #ffffff; + border-radius: 22px; + width : 84px; + height : 35px; + line-height : 35px; + margin-right : 16px; + cursor : pointer; + } + + .next { + + font-size : 14px; + color : #ffffff; + width : 84px; + height : 35px; + background : #2e92f7; + border-radius: 22px; + line-height : 35px; + cursor : pointer; + } + + .luBanLogo { + width : 60px; + height : 57px; + margin-top : -8px; + margin-left: -8px; + } + } + } +} + +// 第五步 +#mask5 { + position : fixed; + top : 0; + right : 0; + left : 0; + z-index : 1003; + display : flex; + flex-direction: column; + align-items : flex-end; + + .selected { + height : 54px; + border : 2px dashed #ffffff; + margin-top : 0.3px; + width : 54px; + margin-right: 5px; + cursor: pointer; + &:hover{ + background-color: #ffffff8c; + } + } + + .arrow { + margin-right: 60px; + } + + .desc { + margin-right: 130px; + font-family : PingFangSC-Regular; + font-size : 14px; + color : #ffffff; + border : 2px dashed #ffffff; + padding : 16px; + width : 200px; + position: relative; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + + &::after{ + content: ""; + width: 10px; + height: 10px; + transform: rotate(45deg); + border-width: 0 2px 2px 0; + border-color: #fff; + border-style: dashed; + position : absolute; + bottom : -7px; + left : 158px; + background-color: #4c4c4c; + } + } + + #confirm5 { + margin-right : 135px; + display : flex; + justify-content: center; + align-items : center; + text-align : center; + padding-top: 27px; + + .skip { + + font-size : 14px; + color : rgba(255, 255, 255, 0.65); + width : 46px; + padding-right: 16px; + cursor : pointer; + } + + .prev { + + font-size : 14px; + color : #ffffff; + border : 1px dashed #ffffff; + border-radius: 22px; + width : 84px; + height : 35px; + line-height : 35px; + margin-right : 16px; + cursor : pointer; + } + + .next { + font-family : PingFangSC-Medium; + font-size : 14px; + color : #ffffff; + width : 84px; + height : 35px; + background : #2e92f7; + border-radius : 22px; + line-height : 35px; + cursor : pointer; + } + + .luBanLogo { + width : 60px; + height : 57px; + margin-top : -8px; + margin-left: -8px; + } + } +} + +// 第六步 +#mask6 { + position : fixed; + right : 0; + top : 54px; + z-index : 1004; + display : flex; + flex-direction: column; + align-items : flex-end; + + .selected { + height : 215px; + border : 2px dashed #ffffff; + margin-top : 0.3px; + width : 118px; + margin-right: 1px; + } + + .arrow { + margin-right: 82px; + } + + .desc { + margin-right: 152px; + font-family : PingFangSC-Regular; + font-size : 14px; + color : #ffffff; + border : 2px dashed #ffffff; + padding : 16px; + width : 200px; + position: relative; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + + &::after{ + content: ""; + width: 10px; + height: 10px; + transform: rotate(45deg); + border-width: 0 2px 2px 0; + border-color: #fff; + border-style: dashed; + position : absolute; + bottom : -7px; + left : 158px; + background-color: #4c4c4c; + } + } + + #confirm6 { + margin-right : 157px; + display : flex; + justify-content: center; + align-items : center; + text-align : center; + padding-top: 27px; + + .skip { + + font-size : 14px; + color : rgba(255, 255, 255, 0.65); + width : 46px; + padding-right: 16px; + cursor : pointer; + } + + .prev { + + font-size : 14px; + color : #ffffff; + border : 1px dashed #ffffff; + border-radius: 22px; + width : 84px; + height : 35px; + line-height : 35px; + margin-right : 16px; + cursor : pointer; + } + + .next { + + font-size : 14px; + color : #ffffff; + width : 84px; + height : 35px; + background : #2e92f7; + border-radius: 22px; + line-height : 35px; + cursor : pointer; + } + + .luBanLogo { + width : 60px; + height : 57px; + margin-top : -5px; + margin-left: -8px; + } + } +} + +// 第七步 +#mask7 { + position : absolute; + right : 27px; + bottom : 103px; + z-index : 1003; + display : flex; + flex-direction: column; + align-items : flex-end; + + .selected { + height : 54px; + border : 2px dashed #ffffff; + margin-right: 0.5px; + width : 65px; + z-index : 1005; + cursor: pointer; + &:hover{ + background-color: #ffffff8c; + } + } + + .item1 { + position: absolute; + + .desc { + margin-left: 97px; + + font-size : 14px; + color : #ffffff; + border : 2px dashed #ffffff; + padding : 16px; + width : 200px; + position: relative; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + + &::after{ + content: ""; + width: 10px; + height: 10px; + transform: rotate(45deg); + border-width: 0 2px 2px 0; + border-color: #fff; + border-style: dashed; + position : absolute; + bottom : -7px; + left : 146px; + background-color: #4c4c4c; + } + } + + #confirm7 { + margin-right : 149px; + display : flex; + justify-content: center; + align-items : center; + text-align : center; + padding-top: 27px; + + .skip { + + font-size : 14px; + color : rgba(255, 255, 255, 0.65); + width : 46px; + padding-right: 16px; + cursor : pointer; + } + + .prev { + + font-size : 14px; + color : #ffffff; + border : 1px dashed #ffffff; + border-radius: 22px; + width : 84px; + height : 35px; + line-height : 35px; + margin-right : 16px; + cursor : pointer; + } + + .next { + + font-size : 14px; + color : #ffffff; + width : 84px; + height : 35px; + background : #2e92f7; + border-radius: 22px; + line-height : 35px; + cursor : pointer; + } + + .luBanLogo { + width : 60px; + height : 57px; + margin-top : -5px; + margin-left: -8px; + } + } + } + + .arrow { + margin-right: 60px; + } + +} + +// 第八步 +#mask8 { + position : fixed; + right : 0; + top : 50px; + z-index : 1003; + display : flex; + pointer-events: none; + + .selected { + border : 2px dashed #ffffff; + margin-right : 0.5px; + height : 100vh; + pointer-events: auto; + width: 655px; + } + + .item1 { + pointer-events: auto; + margin-top : 144px; + position : relative; + + .desc { + margin-left: 84px; + + font-size : 14px; + color : #ffffff; + border : 2px dashed #ffffff; + padding : 16px; + width : 210px; + position: relative; + + span { + color: rgb(9, 140, 234); + font-weight: bold; + } + + &::after{ + content: ""; + width: 10px; + height: 10px; + transform: rotate(45deg); + border-width: 0 2px 2px 0; + border-color: #fff; + border-style: dashed; + position : absolute; + bottom : -7px; + left : 158px; + background-color: #4c4c4c; + } + } + + #confirm8 { + display : flex; + justify-content: center; + align-items : center; + text-align : center; + padding-top: 27px; + padding-right: 12px; + + .skip { + + font-size : 14px; + color : rgba(255, 255, 255, 0.65); + width : 46px; + padding-right: 16px; + cursor : pointer; + } + + .prev { + + font-size : 14px; + color : #ffffff; + border : 1px dashed #ffffff; + border-radius: 22px; + width : 84px; + height : 35px; + line-height : 35px; + margin-right : 16px; + cursor : pointer; + } + + .next { + + font-size : 14px; + color : #ffffff; + width : 84px; + height : 35px; + background : #2e92f7; + border-radius: 22px; + line-height : 35px; + cursor : pointer; + } + + .luBanLogo { + width : 60px; + height : 57px; + margin-top : -5px; + margin-left: -8px; + } + } + } + + .arrow { + margin-top: 65px; + } + +} + +.next { + &:hover { + background: #1C73CB !important; + } + + &:active { + background: #1667B9 !important; + } +} + +.skip { + &:hover { + color: rgba(255, 255, 255, 0.85) !important; + } + + &:active { + color: #FFFFFF !important; + } +} + +.prev { + &:hover { + background: rgba(255, 255, 255, 0.15) !important; + } + + &:active { + background: rgba(255, 255, 255, 0.25) !important; + } +} + +@keyframes rainbow { + 0% { width: 0; } + 100% { width: 405px; } + } + +.luBanLogo { + display: none +} diff --git a/web/packages/dss/module/newGuidance/index.vue b/web/packages/dss/module/newGuidance/index.vue new file mode 100644 index 000000000..1e94e8dde --- /dev/null +++ b/web/packages/dss/module/newGuidance/index.vue @@ -0,0 +1,448 @@ + + + diff --git a/web/packages/dss/package.json b/web/packages/dss/package.json new file mode 100644 index 000000000..f046eeffc --- /dev/null +++ b/web/packages/dss/package.json @@ -0,0 +1,19 @@ +{ + "name": "@dataspherestudio/dss", + "version": "1.1.4", + "dependencies": { + "@dataspherestudio/scriptis": "^1.1.4", + "@dataspherestudio/shared": "^1.1.4" + }, + "devDependencies": { + "@vue/cli-plugin-babel": "3.12.1", + "@vue/cli-plugin-eslint": "3.12.1", + "@vue/cli-service": "3.12.1", + "@vue/eslint-config-standard": "4.0.0" + }, + "scripts": { + "serve": "vue-cli-service serve", + "build": "vue-cli-service build", + "build-sandbox": "vue-cli-service build --mode sandbox" + } +} diff --git a/web/packages/dss/public/favicon.ico b/web/packages/dss/public/favicon.ico new file mode 100644 index 000000000..3fab09991 Binary files /dev/null and b/web/packages/dss/public/favicon.ico differ diff --git a/web/packages/dss/public/index.html b/web/packages/dss/public/index.html new file mode 100644 index 000000000..1dde1c329 --- /dev/null +++ b/web/packages/dss/public/index.html @@ -0,0 +1,34 @@ + + + + + + + + + + + + + +
+ + + diff --git "a/web/packages/dss/public/streamis/Streamis\346\265\201\345\274\217\345\272\224\347\224\250ZIP\345\214\205\344\270\212\344\274\240\346\240\274\345\274\217.txt" "b/web/packages/dss/public/streamis/Streamis\346\265\201\345\274\217\345\272\224\347\224\250ZIP\345\214\205\344\270\212\344\274\240\346\240\274\345\274\217.txt" new file mode 100644 index 000000000..098bf4437 --- /dev/null +++ "b/web/packages/dss/public/streamis/Streamis\346\265\201\345\274\217\345\272\224\347\224\250ZIP\345\214\205\344\270\212\344\274\240\346\240\274\345\274\217.txt" @@ -0,0 +1,64 @@ +流式应用物料包是指的按照Streamis打包规范,将元数据信息(流式应用描述信息),流式应用代码,流式应用使用到的物料等内容打包成zip包。其具体格式如下: + +xxx.zip + ├── meta.json + ├── test.sql + ├── test.jar + ├── file3 + +其中,meta.json格式为: +{ + "projectName": "", # 项目名 + "jobName": "", # 作业名 + "jobType": "flink.sql", # 目前只支持flink.sql、flink.jar + "tags": "", # 应用标签 + "description": "" # 作业描述, + "jobContent": { + # 不同的jobType,其内容各不相同,具体请往下看 + } +} + +如果jobType为"flink.sql",则jobContent为: +{ + "type": "" # file, bml or sql + "sql": "select 1", + "file": "test.sql", + "resourceId": "", + "version": "" +} +其中,如果type为"file",则只识别file字段;如果type为"sql",则只识别sql字段;如果type为"bml",则只识别resourceId和version字段。 + + +如果jobType为"flink.jar",则jobContent为: + +{ + "main.class.jar": "", # string。main class的jar,如:test.jar + "main.class": "", # main class,如 com.webank.Test + "args": "", # main class 的入参,即main函数的args,请以空格为分隔符 + "hdfs.jars"; [], # 依赖的HDFS jars,如:hdfs:///user/enjoyyin/test1.jar + "dependency.jars": [], # 依赖的jars,如:test2.jar + "resources": [] # 依赖的资源文件,如:test.properties +} + + +也支持直接requestBody为JSON的方式创建新Job,或者是更新Job版本,requestBody格式跟meta.json一模一样,只是相关的文件是工程级别的文件。 + + +上传文件: +/api/rest_j/v1/streamis/project/files/upload +{ + "fileName": "", + "version": "", + "projectName": "", + "updateWhenExists": false +} +mutildata带上文件字节流 + +/api/rest_j/v1/streamis/project/files/list +/api/rest_j/v1/streamis/project/files/delete + +/api/rest_j/v1/streamis/job/upload +带上zip + +/api/rest_j/v1/streamis/job/createOrUpdate +requestBody为zip的meta.json \ No newline at end of file diff --git a/web/packages/dss/public/streamis/flinkJar.zip b/web/packages/dss/public/streamis/flinkJar.zip new file mode 100644 index 000000000..80b64c836 Binary files /dev/null and b/web/packages/dss/public/streamis/flinkJar.zip differ diff --git a/web/packages/dss/server/.gitignore b/web/packages/dss/server/.gitignore new file mode 100644 index 000000000..78a0eadae --- /dev/null +++ b/web/packages/dss/server/.gitignore @@ -0,0 +1,15 @@ +.DS_Store +node_modules/ +dist/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* +package-lock.json + +# Editor directories and files +.idea +.vscode +*.suo +*.ntvs* +*.njsproj +*.sln diff --git a/web/packages/dss/server/README.md b/web/packages/dss/server/README.md new file mode 100644 index 000000000..ea48e459a --- /dev/null +++ b/web/packages/dss/server/README.md @@ -0,0 +1,108 @@ +``` + "dependencies": { + "koa": "^2.6.2", + "mysql": "^2.16.0" + } +``` + + +1.设置配置文件 +-- + +``` +// default.js +// 设置配置文件 +const config = { + // 启动端口 + port: 3022, + + // 数据库配置 + database: { + DATABASE: 'ceshi', + USERNAME: 'root', + PASSWORD: '1234', + PORT: '3306', + HOST: 'localhost' + } + } + + module.exports = config + +``` + +2.连接数据库 +-- + +``` +// mysql/index.js + +var mysql = require('mysql'); +var config = require('../config/default.js') + +var pool = mysql.createPool({ + host : config.database.HOST, + user : config.database.USERNAME, + password : config.database.PASSWORD, + database : config.database.DATABASE +}); + + +class Mysql { + constructor () { + + } + query () { + return new Promise((resolve, reject) => { + pool.query('SELECT * from ceshidata', function (error, results, fields) { + if (error) { + throw error + }; + resolve(results) + // console.log('The solution is: ', results[0].solution); + }); + }) + + } +} + +module.exports = new Mysql() + +``` + +3.设置服务器 +-- + +``` +// index.js +const Koa = require('koa') +const config = require('./config/default') +const mysql = require('./mysql') + +const app = new Koa() + +app.use(async (ctx) => { + let data = await mysql.query() + ctx.body = { + "code": 1, + "data": data, + "mesg": 'ok' + } + +}) + +app.listen(config.port) + +console.log(`listening on port ${config.port}`) + +``` +4.启动服务器,去浏览器访问 +-- + +先去数据库添加点数据 + +``` +node index.js +``` +打开浏览器localhost:3022, 然后你就会看到以下数据,自己添加的数据查询出来了 + +![image.png](https://upload-images.jianshu.io/upload_images/1379609-ec8ecf1e60fbf243.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) diff --git a/web/packages/dss/server/config/default.js b/web/packages/dss/server/config/default.js new file mode 100644 index 000000000..d4172a84b --- /dev/null +++ b/web/packages/dss/server/config/default.js @@ -0,0 +1,35 @@ +const config = { + // 启动端口 + port: 3022, + + // 数据库配置 + dev: { + database: { + DATABASE: '', + USERNAME: '', + PASSWORD: '', + PORT: '', + HOST: '' + } + }, + test: { + database: { + DATABASE: '', + USERNAME: '', + PASSWORD: '', + PORT: '', + HOST: '' + } + }, + prd: { + database: { + DATABASE: '', + USERNAME: '', + PASSWORD: '', + PORT: '', + HOST: '' + } + } +} + +module.exports = config diff --git a/web/packages/dss/server/index.js b/web/packages/dss/server/index.js new file mode 100644 index 000000000..de4864999 --- /dev/null +++ b/web/packages/dss/server/index.js @@ -0,0 +1,55 @@ +const Koa = require('koa') +const url = require('url'); +const Router = require('koa-router') +const config = require('./config/default') +const mysql = require('./mysql') +const bodyParser = require('koa-body') +const router = new Router({ + prefix: '/application' +}) + +const app = new Koa() + +// app.use(json()) +app.use(async (ctx, next) => { + await next() +}) + +router.get('/query', async (ctx, next) => { + let id = url.parse(ctx.request.url, true).query.id + let data = id ? await mysql.query(id) : await mysql.queryAll() + ctx.response.body = { + "code": 0, + "status": 0, + "data": data, + "message": "success" + } +}) + +router.post('/update', async (ctx, next) => { + let data = await mysql.createApplication(ctx.request.body) + ctx.response.body = { + "code": 0, + "status": 0, + "message": "success" + } +}) + +router.get('/getMenu', async (ctx, next) => { + let data = await mysql.queryMenu() + ctx.response.body = { + "code": 0, + "status": 0, + "data": data, + "message": "success" + } +}) + +app.use(bodyParser({ + multipart: true // 支持表单解析 +})) +app.use(router.routes()) + +app.listen(config.port) + +console.log(`listening on port ${config.port}`) diff --git a/web/packages/dss/server/mysql/index.js b/web/packages/dss/server/mysql/index.js new file mode 100644 index 000000000..adb5ca025 --- /dev/null +++ b/web/packages/dss/server/mysql/index.js @@ -0,0 +1,147 @@ +var mysql = require("mysql"); +var config = require("../config/default.js"); + +var env = process.env.NODE_ENV; +env = env ? env : "dev"; + +var pool = mysql.createPool({ + host: config[env].database.HOST, + user: config[env].database.USERNAME, + password: config[env].database.PASSWORD, + database: config[env].database.DATABASE, + port: config[env].database.PORT +}); + +class Mysql { + constructor() {} + queryMenu() { + return new Promise((resolve, reject) => { + pool.query("SELECT * from dss_onestop_menu a ORDER BY a.id;", function( + error, + results, + fields + ) { + if (error) { + throw error; + } + resolve(results); + }); + }); + } + queryAll() { + return new Promise((resolve, reject) => { + pool.query( + "SELECT * from dss_onestop_menu_application b RIGHT JOIN dss_application a ON a.id = b.application_id ORDER BY a.id;", + function(error, results, fields) { + if (error) { + throw error; + } + resolve(results); + } + ); + }); + } + query(id) { + return new Promise((resolve, reject) => { + pool.query( + `SELECT * from dss_onestop_menu_application b RIGHT JOIN dss_application a ON a.id = b.application_id WHERE a.id=${id};`, + function(error, results, fields) { + if (error) { + throw error; + } + resolve(results); + } + ); + }); + } + createApplication(data) { + return new Promise((resolve, reject) => { + if ( + !data.title_en || + !data.title_cn || + !data.url || + !data.onestop_menu_id || + !data.project_url || + !data.homepage_url + ) { + throw new Error("必填字段缺损"); + } + if (!data.id) { + let sql = `INSERT INTO dss_application(id,name,url,project_url,if_iframe,homepage_url,redirect_url) VALUES(0,?,?,?,?,?,?)`; + let params = [ + data.title_en, + data.url, + data.project_url, + data.if_iframe, + data.homepage_url, + data.redirect_url + ]; + pool.query(sql, params, function(error, result) { + if (error) { + throw error; + } + if (result && result.insertId) { + let sql2 = `INSERT INTO dss_onestop_menu_application(id,application_id,onestop_menu_id,title_en,title_cn,desc_en,desc_cn,labels_en,labels_cn,is_active,access_button_en,access_button_cn) VALUES(0,?,?,?,?,?,?,?,?,?,?,?)`; + let params2 = [ + result.insertId, + data.onestop_menu_id, + data.title_en, + data.title_cn, + data.desc_en, + data.desc_cn, + data.labels_en, + data.labels_cn, + data.is_active, + data.access_button_en || `Enter ${data.title_en}`, + data.access_button_cn || `进入 ${data.title_cn}` + ]; + pool.query(sql2, params2, function(error2, result2) { + if (error2) { + throw error2; + } + resolve(result2); + }); + } + }); + } else { + let sql = `UPDATE dss_application SET name=?,url=?,project_url=?,if_iframe=?,homepage_url=?,redirect_url=? WHERE id=?`; + let params = [ + data.title_en, + data.url, + data.project_url, + data.if_iframe, + data.homepage_url, + data.redirect_url, + data.id + ]; + pool.query(sql, params, function(error, result) { + if (error) { + throw error; + } + let sql2 = `UPDATE dss_onestop_menu_application SET onestop_menu_id=?,title_en=?,title_cn=?,desc_en=?,desc_cn=?,labels_en=?,labels_cn=?,is_active=?,access_button_en=?,access_button_cn=? WHERE application_id=?`; + let params2 = [ + data.onestop_menu_id, + data.title_en, + data.title_cn, + data.desc_en, + data.desc_cn, + data.labels_en, + data.labels_cn, + data.is_active, + data.access_button_en || `Enter ${data.title_en}`, + data.access_button_cn || `进入 ${data.title_cn}`, + data.id + ]; + pool.query(sql2, params2, function(error2, result2) { + if (error2) { + throw error2; + } + resolve(result2); + }); + }); + } + }); + } +} + +module.exports = new Mysql(); diff --git a/web/packages/dss/server/package.json b/web/packages/dss/server/package.json new file mode 100644 index 000000000..99d111e16 --- /dev/null +++ b/web/packages/dss/server/package.json @@ -0,0 +1,20 @@ +{ + "name": "koa-mayql", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "dev": "cross-env NODE_ENV=dev node index.js", + "test": "cross-env NODE_ENV=test node index.js", + "prd": "cross-env NODE_ENV=prd node index.js" + }, + "author": "", + "license": "ISC", + "dependencies": { + "cross-env": "^7.0.3", + "koa": "^2.6.2", + "koa-body": "^4.2.0", + "koa-router": "^10.0.0", + "mysql": "^2.16.0" + } +} diff --git a/web/packages/dss/server/yarn.lock b/web/packages/dss/server/yarn.lock new file mode 100644 index 000000000..96061b072 --- /dev/null +++ b/web/packages/dss/server/yarn.lock @@ -0,0 +1,293 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +accepts@^1.3.5: + version "1.3.5" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.5.tgz#eb777df6011723a3b14e8a72c0805c8e86746bd2" + dependencies: + mime-types "~2.1.18" + negotiator "0.6.1" + +any-promise@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" + +bignumber.js@4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-4.1.0.tgz#db6f14067c140bd46624815a7916c92d9b6c24b1" + +cache-content-type@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/cache-content-type/-/cache-content-type-1.0.1.tgz#035cde2b08ee2129f4a8315ea8f00a00dba1453c" + dependencies: + mime-types "^2.1.18" + ylru "^1.2.0" + +co@^4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" + +content-disposition@~0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4" + +content-type@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" + +cookies@~0.7.1: + version "0.7.3" + resolved "https://registry.yarnpkg.com/cookies/-/cookies-0.7.3.tgz#7912ce21fbf2e8c2da70cf1c3f351aecf59dadfa" + dependencies: + depd "~1.1.2" + keygrip "~1.0.3" + +core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + +debug@~3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + dependencies: + ms "2.0.0" + +deep-equal@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" + +delegates@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" + +depd@^1.1.2, depd@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" + +destroy@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" + +ee-first@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.2.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" + +error-inject@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37" + +escape-html@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" + +fresh@~0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" + +http-assert@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/http-assert/-/http-assert-1.4.0.tgz#0e550b4fca6adf121bbeed83248c17e62f593a9a" + dependencies: + deep-equal "~1.0.1" + http-errors "~1.7.1" + +http-errors@^1.6.3, http-errors@~1.7.1: + version "1.7.1" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.1.tgz#6a4ffe5d35188e1c39f872534690585852e1f027" + dependencies: + depd "~1.1.2" + inherits "2.0.3" + setprototypeof "1.1.0" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + +inherits@2.0.3, inherits@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + +is-generator-function@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/is-generator-function/-/is-generator-function-1.0.7.tgz#d2132e529bb0000a7f80794d4bdf5cd5e5813522" + +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +json-stringify-safe@5: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + +keygrip@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.0.3.tgz#399d709f0aed2bab0a059e0cdd3a5023a053e1dc" + +koa-compose@^3.0.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/koa-compose/-/koa-compose-3.2.1.tgz#a85ccb40b7d986d8e5a345b3a1ace8eabcf54de7" + dependencies: + any-promise "^1.1.0" + +koa-compose@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/koa-compose/-/koa-compose-4.1.0.tgz#507306b9371901db41121c812e923d0d67d3e877" + +koa-convert@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/koa-convert/-/koa-convert-1.2.0.tgz#da40875df49de0539098d1700b50820cebcd21d0" + dependencies: + co "^4.6.0" + koa-compose "^3.0.0" + +koa-is-json@1, koa-is-json@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/koa-is-json/-/koa-is-json-1.0.0.tgz#273c07edcdcb8df6a2c1ab7d59ee76491451ec14" + +koa-json@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/koa-json/-/koa-json-2.0.2.tgz#36af14e6ea1f5d646d7c44a285701c6f85a4fde4" + dependencies: + koa-is-json "1" + streaming-json-stringify "3" + +koa@^2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/koa/-/koa-2.6.2.tgz#57ba4d049b0a99cae0d594e6144e2931949a7ce1" + dependencies: + accepts "^1.3.5" + cache-content-type "^1.0.0" + content-disposition "~0.5.2" + content-type "^1.0.4" + cookies "~0.7.1" + debug "~3.1.0" + delegates "^1.0.0" + depd "^1.1.2" + destroy "^1.0.4" + error-inject "^1.0.0" + escape-html "^1.0.3" + fresh "~0.5.2" + http-assert "^1.3.0" + http-errors "^1.6.3" + is-generator-function "^1.0.7" + koa-compose "^4.1.0" + koa-convert "^1.2.0" + koa-is-json "^1.0.0" + on-finished "^2.3.0" + only "~0.0.2" + parseurl "^1.3.2" + statuses "^1.5.0" + type-is "^1.6.16" + vary "^1.1.2" + +media-typer@0.3.0: + version "0.3.0" + resolved "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" + +mime-db@~1.37.0: + version "1.37.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" + +mime-types@^2.1.18, mime-types@~2.1.18: + version "2.1.21" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" + dependencies: + mime-db "~1.37.0" + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + +mysql@^2.16.0: + version "2.16.0" + resolved "https://registry.yarnpkg.com/mysql/-/mysql-2.16.0.tgz#b23b22ab5de44fc2d5d32bd4f5af6653fc45e2ba" + dependencies: + bignumber.js "4.1.0" + readable-stream "2.3.6" + safe-buffer "5.1.2" + sqlstring "2.3.1" + +negotiator@0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" + +on-finished@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" + dependencies: + ee-first "1.1.2" + +only@~0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/only/-/only-0.0.2.tgz#2afde84d03e50b9a8edc444e30610a70295edfb4" + +parseurl@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" + +process-nextick-args@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + +readable-stream@2, readable-stream@2.3.6: + version "2.3.6" + resolved "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.2" + util-deprecate "~1.0.1" + +safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + +setprototypeof@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" + +sqlstring@2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40" + +"statuses@>= 1.5.0 < 2", statuses@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" + +streaming-json-stringify@3: + version "3.1.0" + resolved "https://registry.yarnpkg.com/streaming-json-stringify/-/streaming-json-stringify-3.1.0.tgz#80200437a993cc39c4fe00263b7b3b903ac87af5" + dependencies: + json-stringify-safe "5" + readable-stream "2" + +string_decoder@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.2.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + dependencies: + safe-buffer "~5.1.0" + +toidentifier@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" + +type-is@^1.6.16: + version "1.6.16" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.16.tgz#f89ce341541c672b25ee7ae3c73dee3b2be50194" + dependencies: + media-typer "0.3.0" + mime-types "~2.1.18" + +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + +vary@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" + +ylru@^1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ylru/-/ylru-1.2.1.tgz#f576b63341547989c1de7ba288760923b27fe84f" diff --git a/web/packages/dss/src/common-router.js b/web/packages/dss/src/common-router.js new file mode 100644 index 000000000..57bebd860 --- /dev/null +++ b/web/packages/dss/src/common-router.js @@ -0,0 +1,110 @@ +export const subAppRoutes = { + path: '', + name: 'layout', + component: () => import('../view/layout.vue'), + redirect: '/login', + meta: { + publicPage: true, // 权限公开 + }, + children: [{ + path: '/commonIframe/:appName', + name: 'commonIframe', + meta: { + publicPage: true, + }, + component: () => + import('../view/commonIframe/index.vue'), + }, + // 新增一个redirect路由mock路由刷新效果,放在这里共享layout可保持header不会unmount,减少ajax + { + path: '/redirect/:path(.*)', + meta: { + publicPage: true, + }, + component: () => import('../view/redirect/index.vue') + }] +} + +export default [ + // 日志查看 + { + path: '/log', + name: 'log', + meta: { + title: 'Log', + publicPage: true, + }, + component: () => + import('../view/logPage/index.vue') + }, + { + path: '/login', + name: 'login', + meta: { + title: 'Login', + publicPage: true, + }, + component: () => + import('../view/login/index.vue'), + }, + // 公用页面,不受权限控制 + { + path: '/500', + name: 'serverErrorPage', + meta: { + title: '服务器错误', + publicPage: true, + }, + component: () => + import('../view/500.vue'), + }, + { + path: '/404', + name: 'pageNotFound', + meta: { + title: '404', + publicPage: true, + }, + component: () => + import('../view/404.vue'), + }, + { + path: '/403', + name: 'pageForbidden', + meta: { + title: '403', + publicPage: true, + }, + component: () => + import('../view/403.vue'), + }, + { + path: '/vue-process-demo', + name: 'vueprocess', + meta: { + title: 'vueprocess', + publicPage: true, + }, + component: () => + import('../view/vue-process-demo/index.vue'), + }, + // svg可用图标预览 + { + path: '/icon', + name: 'icon', + meta: { + title: 'icon', + publicPage: true, + }, + component: () => + import('../view/icon.vue'), + }, + { + path: '*', + meta: { + publicPage: true, + }, + component: () => + import('../view/404.vue'), + } +] diff --git a/web/packages/dss/src/dynamic-apps.js b/web/packages/dss/src/dynamic-apps.js new file mode 100644 index 000000000..3ee451027 --- /dev/null +++ b/web/packages/dss/src/dynamic-apps.js @@ -0,0 +1,53 @@ +const apps = require('dynamic-modules') +console.log(apps, 'apps') +window.$APP_CONF = apps.conf + +import { merge } from 'lodash' +import { subAppRoutes } from "./common-router" +// 根据module参数配置要打包的应用,生成的虚拟模块 +// 子应用可以有独立的顶层路由配置layout,header,footer,可配置micro_module参数使用子应用layout + +let subRoutes = subAppRoutes +const appsRoutes = Object.values(apps.appsRoutes) + +/** + * * 科管版本:scriptis + 管理台(linkis已从dss拆出去) + * * npm run build --module=scriptis --micro_module=scriptis + * * 数据服务子应用:apiServices和workspace一起打包 + * * npm run build --module=apiServices,workspace --micro_module=apiServices + * ! micro_module参数要和module里值一样,否则找不到 + */ +if (apps.microModule) { + if (apps.appsRoutes[apps.microModule].subAppRoutes) { + subRoutes = apps.appsRoutes[apps.microModule].subAppRoutes + } +} + +if (appsRoutes) { + appsRoutes.forEach(route => { + if (apps.microModule === 'apiServices' && route.apiServicesRoutes) { + subRoutes.children = subRoutes.children.concat(route.apiServicesRoutes) + } else { + subRoutes.children = subRoutes.children.concat(route.default) + } + }); +} + +// 合并公共国际化 +const i18nConfig = { + 'en': require('@dataspherestudio/shared/common/i18n/en.json'), + 'zh-CN': require('@dataspherestudio/shared/common/i18n/zh.json') +} + +// 处理国际化 +if (apps.appsI18n) { + apps.appsI18n.forEach(item => { + merge(i18nConfig, item) + }); +} + +export { + subRoutes, + i18nConfig, + apps +} diff --git a/web/packages/dss/src/main.js b/web/packages/dss/src/main.js new file mode 100644 index 000000000..09b305494 --- /dev/null +++ b/web/packages/dss/src/main.js @@ -0,0 +1,92 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import Vue from 'vue' +import iView from 'iview' +import VueRouter from 'vue-router' + +import { apps, i18nConfig } from './dynamic-apps' +import i18n from '@dataspherestudio/shared/common/i18n' + +i18n.mergeLocaleMessage(Vue.config.lang, i18nConfig[Vue.config.lang]) + +import component from '@dataspherestudio/shared/components' +import App from '../view/app.vue' +import router from './router' + +import mixinDispatch from '@dataspherestudio/shared/common/service/moduleMixin' +import plugin from '@dataspherestudio/shared/common/util/plugin' + +// 导入各模块的公共路径常量 +import API_PATH from '@dataspherestudio/shared/common/config/apiPath.js' +import 'iview/dist/styles/iview.css' +import '@dataspherestudio/shared/common/style/theme/default.less' + +// Icon +import '@dataspherestudio/shared/components/svgIcon/index.js' +import '../module/index.js' + +// 扩展模块 +if (apps.exts) { + plugin.init(apps.exts) +} + +// moduleMixin +if (apps.requireComponent) { + apps.requireComponent.forEach(item=>{ + mixinDispatch(item) + }) +} +if (apps.requireComponentVue) { + apps.requireComponentVue.forEach(item=>{ + mixinDispatch(undefined, item) + }) +} + +Vue.use(VueRouter) +Vue.use(component) + +Vue.use(iView, { + i18n: (key, value) => i18n.t(key, value) +}) + +Vue.config.productionTip = false +Vue.prototype.$Message.config({ + duration: 3 +}) +Vue.prototype.$Notice.config({ + duration: 4 +}) + +// 全局变量 +Vue.prototype.$API_PATH = API_PATH; +Vue.prototype.$APP_CONF = apps.conf || {}; + +new Vue({ + router, + i18n, + render: (h) => h(App) +}).$mount('#app') + +console.log(`当前环境:${process.env.NODE_ENV}`) + +if (localStorage.getItem('theme') === 'dark') { + window.document.documentElement.setAttribute('data-theme', 'dark'); +} else { + window.document.documentElement.setAttribute('data-theme', ''); +} + diff --git a/web/packages/dss/src/router.js b/web/packages/dss/src/router.js new file mode 100644 index 000000000..cc0b81f11 --- /dev/null +++ b/web/packages/dss/src/router.js @@ -0,0 +1,68 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import VueRouter from "vue-router"; +import routes from "./common-router" +import { apps, subRoutes } from './dynamic-apps' +import storage from '@dataspherestudio/shared/common/helper/storage'; + +// 解决重复点击路由跳转报错 +const originalPush = VueRouter.prototype.push; +VueRouter.prototype.push = function push(location) { + return originalPush.call(this, location).catch(err => err) +} + +routes.unshift(subRoutes) + +const router = new VueRouter({ + routes +}); + +router.beforeEach((to, from, next) => { + if (to.meta) { + // 给路由添加参数,控制显示对应header + if (to.meta.header) { + to.query.showHeader = to.meta.header + } + // 路由控制不需要显示 footer + if (to.meta.noFooter) { + to.query.noFooter = to.meta.noFooter; + } + if (to.meta.admin) { + // 如果不是管理员权限,则跳转404,防止手动修改路由跳转 + const baseInfo = storage.get("baseInfo", 'local'); + if (baseInfo && baseInfo.isAdmin) { + next(); + } else { + next('/404') + } + } else if (to.meta.publicPage) { + // 公共页面不需要权限控制(404,500) + next(); + } else { + next() + } + } +}); + +router.afterEach((to) => { + if (to.meta) { + document.title = to.meta.title || (apps.conf ? apps.conf.app_name || '' : ''); + } +}); + +export default router; diff --git a/web/packages/dss/view/403.vue b/web/packages/dss/view/403.vue new file mode 100644 index 000000000..a63e424ff --- /dev/null +++ b/web/packages/dss/view/403.vue @@ -0,0 +1,6 @@ + diff --git a/web/packages/dss/view/404.vue b/web/packages/dss/view/404.vue new file mode 100644 index 000000000..949075006 --- /dev/null +++ b/web/packages/dss/view/404.vue @@ -0,0 +1,14 @@ + + diff --git a/web/packages/dss/view/500.vue b/web/packages/dss/view/500.vue new file mode 100644 index 000000000..4a1ce9d72 --- /dev/null +++ b/web/packages/dss/view/500.vue @@ -0,0 +1,23 @@ + + diff --git a/web/packages/dss/view/app.vue b/web/packages/dss/view/app.vue new file mode 100644 index 000000000..a8accd073 --- /dev/null +++ b/web/packages/dss/view/app.vue @@ -0,0 +1,58 @@ + + + diff --git a/web/packages/dss/view/commonIframe/index.vue b/web/packages/dss/view/commonIframe/index.vue new file mode 100644 index 000000000..4082b8a5e --- /dev/null +++ b/web/packages/dss/view/commonIframe/index.vue @@ -0,0 +1,74 @@ + + + + + + + + + diff --git a/web/packages/workflows/module/process/component/arguments.vue b/web/packages/workflows/module/process/component/arguments.vue new file mode 100644 index 000000000..58fb33ed5 --- /dev/null +++ b/web/packages/workflows/module/process/component/arguments.vue @@ -0,0 +1,188 @@ + + + diff --git a/web/packages/workflows/module/process/component/associateScript.vue b/web/packages/workflows/module/process/component/associateScript.vue new file mode 100644 index 000000000..afc5b793d --- /dev/null +++ b/web/packages/workflows/module/process/component/associateScript.vue @@ -0,0 +1,178 @@ + + + + diff --git a/web/packages/workflows/module/process/component/console.vue b/web/packages/workflows/module/process/component/console.vue new file mode 100644 index 000000000..5ae507f5a --- /dev/null +++ b/web/packages/workflows/module/process/component/console.vue @@ -0,0 +1,627 @@ + + + diff --git a/web/packages/workflows/module/process/component/modal.vue b/web/packages/workflows/module/process/component/modal.vue new file mode 100644 index 000000000..6f2a238b1 --- /dev/null +++ b/web/packages/workflows/module/process/component/modal.vue @@ -0,0 +1,136 @@ + + + + diff --git a/web/packages/workflows/module/process/component/nodeparameter.vue b/web/packages/workflows/module/process/component/nodeparameter.vue new file mode 100644 index 000000000..d1bc7d7f8 --- /dev/null +++ b/web/packages/workflows/module/process/component/nodeparameter.vue @@ -0,0 +1,447 @@ + + diff --git a/web/packages/workflows/module/process/component/resource.vue b/web/packages/workflows/module/process/component/resource.vue new file mode 100644 index 000000000..29f999d7b --- /dev/null +++ b/web/packages/workflows/module/process/component/resource.vue @@ -0,0 +1,195 @@ + + + diff --git a/web/packages/workflows/module/process/images/SparkSQL.svg b/web/packages/workflows/module/process/images/SparkSQL.svg new file mode 100644 index 000000000..254f1496b --- /dev/null +++ b/web/packages/workflows/module/process/images/SparkSQL.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/dashboard.svg b/web/packages/workflows/module/process/images/dashboard.svg new file mode 100644 index 000000000..c878af0a2 --- /dev/null +++ b/web/packages/workflows/module/process/images/dashboard.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/datacheck.svg b/web/packages/workflows/module/process/images/datacheck.svg new file mode 100644 index 000000000..5451e9a21 --- /dev/null +++ b/web/packages/workflows/module/process/images/datacheck.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/display.svg b/web/packages/workflows/module/process/images/display.svg new file mode 100644 index 000000000..89736aa29 --- /dev/null +++ b/web/packages/workflows/module/process/images/display.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/email.svg b/web/packages/workflows/module/process/images/email.svg new file mode 100644 index 000000000..b79701b1d --- /dev/null +++ b/web/packages/workflows/module/process/images/email.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/eventchecker.svg b/web/packages/workflows/module/process/images/eventchecker.svg new file mode 100644 index 000000000..0d45f6490 --- /dev/null +++ b/web/packages/workflows/module/process/images/eventchecker.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/fabu.svg b/web/packages/workflows/module/process/images/fabu.svg new file mode 100644 index 000000000..c4e423d4e --- /dev/null +++ b/web/packages/workflows/module/process/images/fabu.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/flow.svg b/web/packages/workflows/module/process/images/flow.svg new file mode 100644 index 000000000..fdc6a3fa1 --- /dev/null +++ b/web/packages/workflows/module/process/images/flow.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/hive.svg b/web/packages/workflows/module/process/images/hive.svg new file mode 100644 index 000000000..be90a2d3a --- /dev/null +++ b/web/packages/workflows/module/process/images/hive.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/lianjie.svg b/web/packages/workflows/module/process/images/lianjie.svg new file mode 100644 index 000000000..1c4c94505 --- /dev/null +++ b/web/packages/workflows/module/process/images/lianjie.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/menu/associate.svg b/web/packages/workflows/module/process/images/menu/associate.svg new file mode 100644 index 000000000..a09256401 --- /dev/null +++ b/web/packages/workflows/module/process/images/menu/associate.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/menu/delete.svg b/web/packages/workflows/module/process/images/menu/delete.svg new file mode 100644 index 000000000..8c6a6309f --- /dev/null +++ b/web/packages/workflows/module/process/images/menu/delete.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/menu/flow.svg b/web/packages/workflows/module/process/images/menu/flow.svg new file mode 100644 index 000000000..1c6120585 --- /dev/null +++ b/web/packages/workflows/module/process/images/menu/flow.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/menu/fuzhi.svg b/web/packages/workflows/module/process/images/menu/fuzhi.svg new file mode 100644 index 000000000..cfba354d4 --- /dev/null +++ b/web/packages/workflows/module/process/images/menu/fuzhi.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/menu/xitongguanlitai.svg b/web/packages/workflows/module/process/images/menu/xitongguanlitai.svg new file mode 100644 index 000000000..c7554dd5b --- /dev/null +++ b/web/packages/workflows/module/process/images/menu/xitongguanlitai.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/menu/zhantie.svg b/web/packages/workflows/module/process/images/menu/zhantie.svg new file mode 100644 index 000000000..f286fad0c --- /dev/null +++ b/web/packages/workflows/module/process/images/menu/zhantie.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newDisplay.svg b/web/packages/workflows/module/process/images/newDisplay.svg new file mode 100644 index 000000000..faaf82250 --- /dev/null +++ b/web/packages/workflows/module/process/images/newDisplay.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/Dashboard.svg b/web/packages/workflows/module/process/images/newIcon/Dashboard.svg new file mode 100644 index 000000000..0de0085a6 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/Dashboard.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/connector.svg b/web/packages/workflows/module/process/images/newIcon/connector.svg new file mode 100644 index 000000000..0cd499d4a --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/connector.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/datacheck.svg b/web/packages/workflows/module/process/images/newIcon/datacheck.svg new file mode 100644 index 000000000..c0005b749 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/datacheck.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/display.svg b/web/packages/workflows/module/process/images/newIcon/display.svg new file mode 100644 index 000000000..29bd75cd9 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/display.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/email.svg b/web/packages/workflows/module/process/images/newIcon/email.svg new file mode 100644 index 000000000..5cdc520f1 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/email.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/eventcheckerf.svg b/web/packages/workflows/module/process/images/newIcon/eventcheckerf.svg new file mode 100644 index 000000000..cf34d920b --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/eventcheckerf.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/eventcheckerw.svg b/web/packages/workflows/module/process/images/newIcon/eventcheckerw.svg new file mode 100644 index 000000000..383b4b882 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/eventcheckerw.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/exchange.svg b/web/packages/workflows/module/process/images/newIcon/exchange.svg new file mode 100644 index 000000000..2580f6817 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/exchange.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/flow.svg b/web/packages/workflows/module/process/images/newIcon/flow.svg new file mode 100644 index 000000000..477af33f4 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/flow.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/hive.svg b/web/packages/workflows/module/process/images/newIcon/hive.svg new file mode 100644 index 000000000..b887b5fd2 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/hive.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/mlss.svg b/web/packages/workflows/module/process/images/newIcon/mlss.svg new file mode 100644 index 000000000..bcb8087e5 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/mlss.svg @@ -0,0 +1 @@ + diff --git a/web/packages/workflows/module/process/images/newIcon/projectNode.svg b/web/packages/workflows/module/process/images/newIcon/projectNode.svg new file mode 100644 index 000000000..10de2b51b --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/projectNode.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/python.svg b/web/packages/workflows/module/process/images/newIcon/python.svg new file mode 100644 index 000000000..78a4e6e47 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/python.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/qualitis.svg b/web/packages/workflows/module/process/images/newIcon/qualitis.svg new file mode 100644 index 000000000..d6d9c33ad --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/qualitis.svg @@ -0,0 +1,3 @@ + + + diff --git a/web/packages/workflows/module/process/images/newIcon/spark.svg b/web/packages/workflows/module/process/images/newIcon/spark.svg new file mode 100644 index 000000000..33f71e5f2 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/spark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/view.svg b/web/packages/workflows/module/process/images/newIcon/view.svg new file mode 100644 index 000000000..29bd75cd9 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/view.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/newIcon/widget.svg b/web/packages/workflows/module/process/images/newIcon/widget.svg new file mode 100644 index 000000000..4b1ea0579 --- /dev/null +++ b/web/packages/workflows/module/process/images/newIcon/widget.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/proj.svg b/web/packages/workflows/module/process/images/proj.svg new file mode 100644 index 000000000..0c7d32259 --- /dev/null +++ b/web/packages/workflows/module/process/images/proj.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/pyspark.svg b/web/packages/workflows/module/process/images/pyspark.svg new file mode 100644 index 000000000..d75542176 --- /dev/null +++ b/web/packages/workflows/module/process/images/pyspark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/scala.svg b/web/packages/workflows/module/process/images/scala.svg new file mode 100644 index 000000000..8f1495c17 --- /dev/null +++ b/web/packages/workflows/module/process/images/scala.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/sender.svg b/web/packages/workflows/module/process/images/sender.svg new file mode 100644 index 000000000..85f13b40c --- /dev/null +++ b/web/packages/workflows/module/process/images/sender.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/shell.svg b/web/packages/workflows/module/process/images/shell.svg new file mode 100644 index 000000000..fddc86fe3 --- /dev/null +++ b/web/packages/workflows/module/process/images/shell.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/tiaodu.svg b/web/packages/workflows/module/process/images/tiaodu.svg new file mode 100644 index 000000000..f5471d1c4 --- /dev/null +++ b/web/packages/workflows/module/process/images/tiaodu.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/tiaodulishi.svg b/web/packages/workflows/module/process/images/tiaodulishi.svg new file mode 100644 index 000000000..65de96676 --- /dev/null +++ b/web/packages/workflows/module/process/images/tiaodulishi.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/view.svg b/web/packages/workflows/module/process/images/view.svg new file mode 100644 index 000000000..89736aa29 --- /dev/null +++ b/web/packages/workflows/module/process/images/view.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/images/workflow/connector.png b/web/packages/workflows/module/process/images/workflow/connector.png new file mode 100644 index 000000000..6338a3a22 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/connector.png differ diff --git a/web/packages/workflows/module/process/images/workflow/dashboard.png b/web/packages/workflows/module/process/images/workflow/dashboard.png new file mode 100644 index 000000000..162f85c07 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/dashboard.png differ diff --git a/web/packages/workflows/module/process/images/workflow/datachecker.png b/web/packages/workflows/module/process/images/workflow/datachecker.png new file mode 100644 index 000000000..38d29c8fe Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/datachecker.png differ diff --git a/web/packages/workflows/module/process/images/workflow/display.png b/web/packages/workflows/module/process/images/workflow/display.png new file mode 100644 index 000000000..5d9eb3bfe Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/display.png differ diff --git a/web/packages/workflows/module/process/images/workflow/eventreceiver.png b/web/packages/workflows/module/process/images/workflow/eventreceiver.png new file mode 100644 index 000000000..5526bc3f0 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/eventreceiver.png differ diff --git a/web/packages/workflows/module/process/images/workflow/eventsender.png b/web/packages/workflows/module/process/images/workflow/eventsender.png new file mode 100644 index 000000000..7493b7449 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/eventsender.png differ diff --git a/web/packages/workflows/module/process/images/workflow/hql.png b/web/packages/workflows/module/process/images/workflow/hql.png new file mode 100644 index 000000000..4c9a78215 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/hql.png differ diff --git a/web/packages/workflows/module/process/images/workflow/mlss.png b/web/packages/workflows/module/process/images/workflow/mlss.png new file mode 100644 index 000000000..f730dbfde Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/mlss.png differ diff --git a/web/packages/workflows/module/process/images/workflow/pyspark.png b/web/packages/workflows/module/process/images/workflow/pyspark.png new file mode 100644 index 000000000..4bebbbf9c Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/pyspark.png differ diff --git a/web/packages/workflows/module/process/images/workflow/python.png b/web/packages/workflows/module/process/images/workflow/python.png new file mode 100644 index 000000000..6aa4b1dcb Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/python.png differ diff --git a/web/packages/workflows/module/process/images/workflow/scala.png b/web/packages/workflows/module/process/images/workflow/scala.png new file mode 100644 index 000000000..8ed0976dc Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/scala.png differ diff --git a/web/packages/workflows/module/process/images/workflow/sendemail.png b/web/packages/workflows/module/process/images/workflow/sendemail.png new file mode 100644 index 000000000..d46950505 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/sendemail.png differ diff --git a/web/packages/workflows/module/process/images/workflow/shell.png b/web/packages/workflows/module/process/images/workflow/shell.png new file mode 100644 index 000000000..16ce40cee Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/shell.png differ diff --git a/web/packages/workflows/module/process/images/workflow/sql.png b/web/packages/workflows/module/process/images/workflow/sql.png new file mode 100644 index 000000000..afb6367f4 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/sql.png differ diff --git a/web/packages/workflows/module/process/images/workflow/subflow.png b/web/packages/workflows/module/process/images/workflow/subflow.png new file mode 100644 index 000000000..767137f54 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/subflow.png differ diff --git a/web/packages/workflows/module/process/images/workflow/widget.png b/web/packages/workflows/module/process/images/workflow/widget.png new file mode 100644 index 000000000..c19bfa149 Binary files /dev/null and b/web/packages/workflows/module/process/images/workflow/widget.png differ diff --git a/web/packages/workflows/module/process/images/xinhaojieshouqi.svg b/web/packages/workflows/module/process/images/xinhaojieshouqi.svg new file mode 100644 index 000000000..45d281570 --- /dev/null +++ b/web/packages/workflows/module/process/images/xinhaojieshouqi.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/packages/workflows/module/process/index.js b/web/packages/workflows/module/process/index.js new file mode 100644 index 000000000..af99c30c6 --- /dev/null +++ b/web/packages/workflows/module/process/index.js @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// import index from './index.vue'; +export default { + // 模块名称 + name: 'Process', + // 规范模块监测什么事件,或者说模块对外提供什么接口 + events: [], // Demo:add + // 规范模块能够触发其他模块什么事件或者说调用其他模块什么接口 + dispatchs: { + WorkSidebar: ['showTree'], + Workbench: ['updateFlowsTab', 'updateFlowsNodeName'], + IndexedDB: ['clearLog', 'clearResult', 'clearProgress', 'updateResult', 'getTree', 'appendTree'], + workflowIndexedDB: [ + 'getNodeCache', 'updateNodeCache', 'removeNodeCache', 'addNodeCache', 'clearNodeCache', 'updateProjectCache', 'getProjectCache' + ], + }, + // 规范模块的动作,由外部调用或者自己执行 + methods: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `${location.protocol}//${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/packages/workflows/module/process/index.scss b/web/packages/workflows/module/process/index.scss new file mode 100644 index 000000000..d0ac3b2d2 --- /dev/null +++ b/web/packages/workflows/module/process/index.scss @@ -0,0 +1,291 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +@import '@dataspherestudio/shared/common/style/variables.scss'; + +.public_loading { + white-space: nowrap; + margin: 0 5px; + + span { + font-size: $font-size-small; + padding: 0 5px; + } +} + +.public-splin-load { + animation: ani-demo-spin 1s linear infinite; +} +@keyframes ani-demo-spin { + from { + transform: rotate(0deg); + } + + 50% { + transform: rotate(180deg); + } + + to { + transform: rotate(360deg); + } +} + +.process-module { + position: relative; + height: 100%; + width: 100%; + + .process-console { + position: absolute; + left: 250px; + bottom: 0; + top: 50%; + z-index: 3; + width: 100%; + transition: all .3s; + } + + .flow-detail { + display: flex; + justify-content: flex-start; + align-items: center; + height: 100%; + + .detail-item { + padding: 0 10px; + color: #515a6e; + font-size: $font-size-base; + } + } + + .button[disabled] { + // background-color: #f3f3f3; + @include bg-color($disabled-bg-color, $dark-disabled-bg-color); + opacity: 1; + cursor: not-allowed !important; + // color: #ccc; + @include font-color(#ccc, $dark-text-desc-color); + + } + + .designer .designer-shape .shape-box { + margin: 2px 0; + padding: 0 10px 0 25px; + } + + .designer .designer-toolbar .devider { + @include border-color(#DEE4EC, $dark-text-color); + } + &.locked { + pointer-events: none; + opacity: .65; + } +} + +.process-module-param { + position: absolute; + top: 0; + bottom: 0; + right: 0; + width: 360px; + height: 100%; + border: $border-width-base $border-style-base $border-color-base; + border-top: 0; + box-shadow: -6px 0 6px -6px $shadow-color; + z-index: 1008; + @include bg-color(#fff, $dark-base-color); + @include border-color($border-color-base, $dark-background-color-header); + overflow-y: auto; + + .process-module-param-modal-header { + height: 36px; + line-height: 36px; + border-bottom: $border-width-base $border-style-base $border-color-base; + @include border-color($border-color-base, $dark-border-color-base); + padding-left: 10px; + position: absolute; + width: 100%; + top: 0; + left: 0; + box-sizing: border-box; + display: flex; + justify-content: flex-start; + align-items: center; + + h3 { + @include font-color($workspace-title-color, $dark-workspace-title-color); + flex: 3; + } + + .save-button { + @include font-color($light-text-color, $dark-text-color); + flex: 2; + text-align: right; + } + + .close-icon { + width: 20px; + height: 20px; + font-weight: 600; + text-align: center; + line-height: 20px; + flex: 1; + cursor: pointer; + @include font-color($light-text-color, $dark-text-color); + &:hover { + // background-color: #ccc; + // @include bg-color(#ccc, $dark-active-menu-item); + } + } + } + + .process-module-param-modal-content { + position: absolute; + top: 40px; + width: 100%; + bottom: 0; + left: 0; + overflow: auto; + + .node-parameter-bar { + padding: 10px 15px 10px 10px; + + .ivu-form-item-error-tip { + text-overflow: -o-ellipsis-lastline; + overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-line-clamp: 2; + line-clamp: 2; + -webkit-box-orient: vertical; + width: 100%; + height: 35px; + } + } + + .node-module-param-modal-header { + @include font-color($workspace-title-color, $dark-workspace-title-color); + padding-left: 10px; + } + + .save-button { + text-align: center; + padding-bottom: 25px; + } + } +} + +.process-module-title { + font-size: 14px; + color: $title-color; +} + +.workflow-arguments { + .workflow-arguments-title-wrap { + @include font-color($workspace-title-color, $dark-workspace-title-color); + display: flex; + justify-content: space-between; + align-items: center; + height: 30px; + padding-right: 7px; + } + + .time-arguments-setting, + .workflow-arguments-setting { + padding: 10px 10px 2px; + margin: 10px; + @include bg-color($menu-dark-subsidiary-color, $dark-base-color); + border-radius: $border-radius-small; + + .time-set-title { + padding-bottom: 5px; + } + + .margin-right { + margin-right: 5px; + margin-bottom: 5px; + } + } +} + +.workflow-resource { + width: 100%; + height: 100%; + position: $relative; + padding: 16px; + + .workflow-resource-list { + .workflow-resource-list-ul { + .workflow-resource-list-item { + height: 28px; + line-height: 28px; + padding: 0 10px; + display: flex; + justify-content: space-between; + align-items: center; + @include bg-color($light-base-color, $dark-base-color); + border-radius: $border-radius-small; + margin: 4px 0; + + &:hover { + background: $background-color-select-hover; + @include bg-color($background-color-select-hover, $dark-active-menu-item); + } + + .workflow-resource-list-name { + max-width: calc(100% - 30px); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + } + } + } + + .workflow-resource-upload { + margin-top: 20px; + } +} + +.we-editor.process-module-editor { + height: 300px; +} + +.repetition-name .ivu-modal-body { + word-break: break-all; +} + +.ivu-modal-confirm-body > div p { + word-break: break-all; +} + +.node-selected { + // background-color: rgba(0, 0, 0, .2) !important; + background: #DEE8F4; + // border: 1px solid #234C7F; + box-shadow: 1px 4px 6px 1px rgba(106,133,167,0.30); + border-radius: 4px; +} + +.ivu-select-dropdown { + max-width: 100%; + overflow: auto; +} + +.schedulisIframe { + width: 100%; + height: 100%; +} diff --git a/web/packages/workflows/module/process/index.vue b/web/packages/workflows/module/process/index.vue new file mode 100755 index 000000000..2b78db0b5 --- /dev/null +++ b/web/packages/workflows/module/process/index.vue @@ -0,0 +1,546 @@ + + + diff --git a/web/packages/workflows/module/process/modal.js b/web/packages/workflows/module/process/modal.js new file mode 100644 index 000000000..ea319a1f0 --- /dev/null +++ b/web/packages/workflows/module/process/modal.js @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * 脚本任务 + */ +export class Script { + /** + * 构造器 + * @param {*} option + */ + constructor(option) { + this.nodeId = option.nodeId; + this.title = option.title; + // 日志 + this.log = option.log || {}; + this.logLine = option.logLine || 1; + // 历史 + this.history = []; + // 进度 + this.progress = { + current: null, + progressInfo: [], + waitingSize: null, + costTime: null, + }; + // 步骤 + this.steps = []; + // 运行结果 + this.result = null; + // 记录结果集的存储路径 + this.resultList = null; + // 当前的运行状态 + this.status = option.status ? option.status : 'Inited'; + // script视图状态 + this.scriptViewState = {}; + } +} diff --git a/web/packages/workflows/module/process/module.vue b/web/packages/workflows/module/process/module.vue new file mode 100755 index 000000000..58c416288 --- /dev/null +++ b/web/packages/workflows/module/process/module.vue @@ -0,0 +1,2618 @@ +