Skip to content

Commit

Permalink
[Improve] openapi request parameter schema improvement (#3916)
Browse files Browse the repository at this point in the history 
* [Improve] remove swagger

* [Improve] REST API improvements

* [Improve] checkstyle issue improvement

* [Improve] class not found fixed

* [Improve] known-dependencies.txt minor improvement

* [Improve] known-dependencies.txt updated

* [Improve] known-dependencies.txt updated

* [Improve] swagger improvement

* [Improve] OpenAPI schema bean improvement

* [Improve] serviceHelper minor improvement

* [Improve] shell script improvement

* [Improve] remove unused plugins dir

* [Improve] shell script minor improvement

* [Improve] openapi improvements

* [Improve] swagger minor improvement

* [Improve] openapi minor improvement

* [Improve] minor improvements

* [Improve] openapi minor improvement

* [Improve] openapi minor improvement

* [Improve] openAPI improvement

* [Improve] openapi.white support

* [Improve] openapi minor improvements
  • Loading branch information
@wolfboys
wolfboys authored Jul 25, 2024
1 parent 616fb3d commit 8b580f1
Show file tree
Hide file tree
Showing 4 changed files with 122 additions and 46 deletions.
2 changes: 2 additions & 0 deletions streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ streampark:
http-auth: 'simple' # default simple, or kerberos
# flink on yarn or spark on yarn, HADOOP_USER_NAME
hadoop-user-name: hdfs
# openapi whitelist,
openapi.white-list:

# flink on yarn or spark on yarn, when the hadoop cluster enable kerberos authentication, it is necessary to set Kerberos authentication parameters.
security:
Expand Down
38 changes: 36 additions & 2 deletions ...ole-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,20 @@
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.DefaultParameterNameDiscoverer;
import org.springframework.core.SpringProperties;
import org.springframework.expression.EvaluationContext;
import org.springframework.expression.Expression;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.expression.spel.support.StandardEvaluationContext;
import org.springframework.stereotype.Component;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;

import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletRequest;

import java.util.HashSet;
import java.util.Set;

@Slf4j
@Component
Expand All @@ -62,9 +71,27 @@ public class StreamParkAspect {
@Autowired
private ApplicationManageService applicationManageService;

private final Set<String> openapiWhitelist = new HashSet<>();

@PostConstruct
public void initOpenapiWhitelist() {
String whiteLists = SpringProperties.getProperty("streampark.openapi.white-list");
if (StringUtils.isNotBlank(whiteLists)) {
String[] whiteList = whiteLists.trim().split("\\s");
for (String order : whiteList) {
if (StringUtils.isNotBlank(order)) {
if (!order.startsWith("/")) {
order = "/" + order;
}
openapiWhitelist.add(order);
}
}
}
}

@Pointcut("execution(public"
+ " org.apache.streampark.console.base.domain.RestResponse"
+ " org.apache.streampark.console.*.controller.*.*(..))")
+ " org.apache.streampark.console.core.controller.*.*(..))")
public void openAPI() {
}

Expand All @@ -77,7 +104,14 @@ public RestResponse openAPI(ProceedingJoinPoint joinPoint) throws Throwable {
if (isApi != null && isApi) {
OpenAPI openAPI = methodSignature.getMethod().getAnnotation(OpenAPI.class);
if (openAPI == null) {
throw new ApiAlertException("current api unsupported!");
HttpServletRequest request =
((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest();
String url = request.getRequestURI();
if (openapiWhitelist.contains(url)) {
log.info("request by openapi white-list: {} ", url);
} else {
throw new ApiAlertException("current api unsupported!");
}
}
}
return (RestResponse) joinPoint.proceed();
Expand Down
2 changes: 2 additions & 0 deletions ...-console-service/src/main/java/org/apache/streampark/console/core/bean/OpenAPISchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ public class OpenAPISchema {

private String url;

private String method;

private List<Schema> header;

private List<Schema> schema;
Expand Down
126 changes: 82 additions & 44 deletions ...-service/src/main/java/org/apache/streampark/console/core/component/OpenAPIComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.streampark.common.util.CURLBuilder;
import org.apache.streampark.common.util.ReflectUtils;
import org.apache.streampark.console.base.util.Tuple2;
import org.apache.streampark.console.core.annotation.OpenAPI;
import org.apache.streampark.console.core.bean.OpenAPISchema;
import org.apache.streampark.console.core.controller.OpenAPIController;
Expand All @@ -29,6 +30,7 @@

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
Expand All @@ -50,6 +52,8 @@ public class OpenAPIComponent {
@Autowired
private AccessTokenService accessTokenService;

private final Map<String, String> types = new HashMap<>();

private final Map<String, OpenAPISchema> schemas = new HashMap<>();

public synchronized OpenAPISchema getOpenAPISchema(String name) {
Expand All @@ -63,20 +67,46 @@ public synchronized OpenAPISchema getOpenAPISchema(String name) {
return schemas.get(name);
}

public String getOpenApiCUrl(String baseUrl, Long appId, Long teamId, String name) {
OpenAPISchema schema = this.getOpenAPISchema(name);
if (schema == null) {
throw new UnsupportedOperationException("Unsupported OpenAPI: " + name);
}

String url = schema.getUrl();
if (StringUtils.isNoneBlank(baseUrl)) {
url = baseUrl + url;
}
CURLBuilder curlBuilder = new CURLBuilder(url);
curlBuilder
.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
.addHeader(
"Authorization",
accessTokenService.getByUserId(ServiceHelper.getUserId()).getToken());

schema.getSchema().forEach(c -> {
if (c.isRequired()) {
if ("appId".equals(c.getBindFor())) {
curlBuilder.addFormData(c.getName(), appId);
} else if ("teamId".equals(c.getBindFor())) {
curlBuilder.addFormData(c.getName(), teamId);
}
} else {
curlBuilder.addFormData(c.getName(), c.getDefaultValue());
}
});
return curlBuilder.build();
}

private void initOpenAPISchema() {
initTypes();
Class<?> clazz = OpenAPIController.class;
RequestMapping requestMapping = clazz.getDeclaredAnnotation(RequestMapping.class);
String basePath = requestMapping.value()[0];
List<Method> methodList = ReflectUtils.getMethodsByAnnotation(clazz, OpenAPI.class);

for (Method method : methodList) {
String[] subUriPath = getMethodUriPath(method);
String subPath = (subUriPath != null && subUriPath.length > 0) ? subUriPath[0] : "";
String restUrl = "/" + basePath;
if (subPath != null) {
restUrl += "/" + subPath;
}
restUrl = restUrl.replaceAll("/+", "/").replaceAll("/$", "");
OpenAPISchema detail = new OpenAPISchema();

List<OpenAPISchema.Schema> headerList = new ArrayList<>();
OpenAPI openAPI = method.getDeclaredAnnotation(OpenAPI.class);
Expand All @@ -89,11 +119,19 @@ private void initOpenAPISchema() {
paramList.add(paramToSchema(param));
}

OpenAPISchema detail = new OpenAPISchema();
detail.setUrl(restUrl);
detail.setSchema(paramList);
detail.setHeader(headerList);

Tuple2<String, String[]> methodURI = getMethodAndRequestURI(method);
String[] requestURI = methodURI.t2;
String uri = (requestURI != null && requestURI.length > 0) ? requestURI[0] : "";
String restURI = "/" + basePath;
if (uri != null) {
restURI += "/" + uri;
}
restURI = restURI.replaceAll("/+", "/").replaceAll("/$", "");
detail.setUrl(restURI);
detail.setMethod(methodURI.t1);
schemas.put(openAPI.name(), detail);
}
}
Expand All @@ -106,71 +144,71 @@ private OpenAPISchema.Schema paramToSchema(OpenAPI.Param param) {
} else {
schema.setBindFor(param.bindFor());
}
schema.setType(param.type().getName());
schema.setRequired(param.required());
schema.setDescription(param.description());
schema.setDefaultValue(param.defaultValue());
String type = types.get(param.type().getSimpleName());
if (type != null) {
schema.setType(type);
} else {
schema.setType("string(" + param.type().getSimpleName() + ")");
}
return schema;
}

private String[] getMethodUriPath(Method method) {
private Tuple2<String, String[]> getMethodAndRequestURI(Method method) {
method.setAccessible(true);

GetMapping getMapping = method.getDeclaredAnnotation(GetMapping.class);
if (getMapping != null) {
return getMapping.value();
return Tuple2.of(HttpMethod.GET.name(), getMapping.value());
}

PostMapping postMapping = method.getDeclaredAnnotation(PostMapping.class);
if (postMapping != null) {
return postMapping.value();
return Tuple2.of(HttpMethod.POST.name(), postMapping.value());
}

DeleteMapping deleteMapping = method.getDeclaredAnnotation(DeleteMapping.class);
if (deleteMapping != null) {
return deleteMapping.value();
return Tuple2.of(HttpMethod.DELETE.name(), deleteMapping.value());
}

PatchMapping patchMapping = method.getDeclaredAnnotation(PatchMapping.class);
if (patchMapping != null) {
return patchMapping.value();
return Tuple2.of(HttpMethod.PATCH.name(), patchMapping.value());
}

PutMapping putMapping = method.getDeclaredAnnotation(PutMapping.class);
if (putMapping != null) {
return putMapping.value();
return Tuple2.of(HttpMethod.PUT.name(), putMapping.value());
}
return null;

throw new IllegalArgumentException("get http method and requestURI failed: " + method.getName());
}

public String getOpenApiCUrl(String baseUrl, Long appId, Long teamId, String name) {
OpenAPISchema schema = this.getOpenAPISchema(name);
if (schema == null) {
throw new UnsupportedOperationException("Unsupported OpenAPI: " + name);
}
private void initTypes() {
types.put("String", "string");

String url = schema.getUrl();
if (StringUtils.isNoneBlank(baseUrl)) {
url = baseUrl + url;
}
CURLBuilder curlBuilder = new CURLBuilder(url);
curlBuilder
.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
.addHeader(
"Authorization",
accessTokenService.getByUserId(ServiceHelper.getUserId()).getToken());
types.put("int", "integer(int32)");
types.put("Integer", "integer(int32)");
types.put("Short", "integer(int32)");

schema.getSchema().forEach(c -> {
if (c.isRequired()) {
if ("appId".equals(c.getBindFor())) {
curlBuilder.addFormData(c.getName(), appId);
} else if ("teamId".equals(c.getBindFor())) {
curlBuilder.addFormData(c.getName(), teamId);
}
} else {
curlBuilder.addFormData(c.getName(), c.getDefaultValue());
}
});
return curlBuilder.build();
types.put("long", "integer(int64)");
types.put("Long", "integer(int64)");

types.put("double", "number(double)");
types.put("Double", "number(double)");

types.put("float", "number(float)");
types.put("Float", "number(float)");
types.put("boolean", "boolean");
types.put("Boolean", "boolean");

types.put("byte", "string(byte)");
types.put("Byte", "string(byte)");

types.put("Date", "string(date)");
types.put("DateTime", "string(datetime)");
}
}

0 comments on commit 8b580f1

Please sign in to comment.