From 27e34ecad002ebe07e0ae61959f2c5b17eaf68d8 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Tue, 24 Dec 2024 11:24:30 +0800 Subject: [PATCH] [CELEBORN-1797] Support to adjust the logger level with RESTful API during runtime ### What changes were proposed in this pull request? Support to adjust the logger level during runtime without restarting the server. ### Why are the changes needed? It is useful for debug, likes hadoop daemonlog command: https://hadoop.apache.org/docs/r3.4.1/hadoop-project-dist/hadoop-common/CommandsManual.html#daemonlog ### Does this PR introduce _any_ user-facing change? Yes, new RESTful api. ### How was this patch tested? GA. image image image image Closes #3022 from turboFei/log_level. Lead-authored-by: Wang, Fei Co-authored-by: Fei Wang Signed-off-by: mingji --- .../celeborn/rest/v1/master/LoggerApi.java | 233 ++++++++++++++++++ .../celeborn/rest/v1/model/LoggerInfo.java | 139 +++++++++++ .../celeborn/rest/v1/model/LoggerInfos.java | 120 +++++++++ .../celeborn/rest/v1/worker/LoggerApi.java | 233 ++++++++++++++++++ .../src/main/openapi3/master_rest_v1.yaml | 67 +++++ .../src/main/openapi3/worker_rest_v1.yaml | 67 +++++ project/CelebornBuild.scala | 4 + service/pom.xml | 10 + .../http/api/v1/ApiV1BaseResource.scala | 3 + .../common/http/api/v1/LoggerResource.scala | 88 +++++++ .../http/api/v1/ApiV1BaseResourceSuite.scala | 82 +++++- 11 files changed, 1045 insertions(+), 1 deletion(-) create mode 100644 openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/master/LoggerApi.java create mode 100644 openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/model/LoggerInfo.java create mode 100644 openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/model/LoggerInfos.java create mode 100644 openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/worker/LoggerApi.java create mode 100644 service/src/main/scala/org/apache/celeborn/server/common/http/api/v1/LoggerResource.scala diff --git a/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/master/LoggerApi.java b/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/master/LoggerApi.java new file mode 100644 index 00000000000..9586d25df90 --- /dev/null +++ b/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/master/LoggerApi.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.celeborn.rest.v1.master; + +import com.fasterxml.jackson.core.type.TypeReference; + +import org.apache.celeborn.rest.v1.master.invoker.ApiException; +import org.apache.celeborn.rest.v1.master.invoker.ApiClient; +import org.apache.celeborn.rest.v1.master.invoker.BaseApi; +import org.apache.celeborn.rest.v1.master.invoker.Configuration; +import org.apache.celeborn.rest.v1.master.invoker.Pair; + +import org.apache.celeborn.rest.v1.model.HandleResponse; +import org.apache.celeborn.rest.v1.model.LoggerInfo; +import org.apache.celeborn.rest.v1.model.LoggerInfos; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; + +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", comments = "Generator version: 7.8.0") +public class LoggerApi extends BaseApi { + + public LoggerApi() { + super(Configuration.getDefaultApiClient()); + } + + public LoggerApi(ApiClient apiClient) { + super(apiClient); + } + + /** + * + * Get the logger level, return all loggers if no name specified. + * @param name The logger name. (optional) + * @param all Return all logger instances if true, otherwise return all configured loggers. (optional, default to false) + * @return LoggerInfos + * @throws ApiException if fails to make API call + */ + public LoggerInfos getLogger(String name, Boolean all) throws ApiException { + return this.getLogger(name, all, Collections.emptyMap()); + } + + + /** + * + * Get the logger level, return all loggers if no name specified. + * @param name The logger name. (optional) + * @param all Return all logger instances if true, otherwise return all configured loggers. (optional, default to false) + * @param additionalHeaders additionalHeaders for this call + * @return LoggerInfos + * @throws ApiException if fails to make API call + */ + public LoggerInfos getLogger(String name, Boolean all, Map additionalHeaders) throws ApiException { + Object localVarPostBody = null; + + // create path and map variables + String localVarPath = "/api/v1/loggers"; + + StringJoiner localVarQueryStringJoiner = new StringJoiner("&"); + String localVarQueryParameterBaseName; + List localVarQueryParams = new ArrayList(); + List localVarCollectionQueryParams = new ArrayList(); + Map localVarHeaderParams = new HashMap(); + Map localVarCookieParams = new HashMap(); + Map localVarFormParams = new HashMap(); + + localVarQueryParams.addAll(apiClient.parameterToPair("name", name)); + localVarQueryParams.addAll(apiClient.parameterToPair("all", all)); + + localVarHeaderParams.putAll(additionalHeaders); + + + + final String[] localVarAccepts = { + "application/json" + }; + final String localVarAccept = apiClient.selectHeaderAccept(localVarAccepts); + + final String[] localVarContentTypes = { + + }; + final String localVarContentType = apiClient.selectHeaderContentType(localVarContentTypes); + + String[] localVarAuthNames = new String[] { "basic" }; + + TypeReference localVarReturnType = new TypeReference() {}; + return apiClient.invokeAPI( + localVarPath, + "GET", + localVarQueryParams, + localVarCollectionQueryParams, + localVarQueryStringJoiner.toString(), + localVarPostBody, + localVarHeaderParams, + localVarCookieParams, + localVarFormParams, + localVarAccept, + localVarContentType, + localVarAuthNames, + localVarReturnType + ); + } + + /** + * + * Set the logger level. + * @param loggerInfo (optional) + * @return HandleResponse + * @throws ApiException if fails to make API call + */ + public HandleResponse setLogger(LoggerInfo loggerInfo) throws ApiException { + return this.setLogger(loggerInfo, Collections.emptyMap()); + } + + + /** + * + * Set the logger level. + * @param loggerInfo (optional) + * @param additionalHeaders additionalHeaders for this call + * @return HandleResponse + * @throws ApiException if fails to make API call + */ + public HandleResponse setLogger(LoggerInfo loggerInfo, Map additionalHeaders) throws ApiException { + Object localVarPostBody = loggerInfo; + + // create path and map variables + String localVarPath = "/api/v1/loggers"; + + StringJoiner localVarQueryStringJoiner = new StringJoiner("&"); + String localVarQueryParameterBaseName; + List localVarQueryParams = new ArrayList(); + List localVarCollectionQueryParams = new ArrayList(); + Map localVarHeaderParams = new HashMap(); + Map localVarCookieParams = new HashMap(); + Map localVarFormParams = new HashMap(); + + + localVarHeaderParams.putAll(additionalHeaders); + + + + final String[] localVarAccepts = { + "application/json" + }; + final String localVarAccept = apiClient.selectHeaderAccept(localVarAccepts); + + final String[] localVarContentTypes = { + "application/json" + }; + final String localVarContentType = apiClient.selectHeaderContentType(localVarContentTypes); + + String[] localVarAuthNames = new String[] { "basic" }; + + TypeReference localVarReturnType = new TypeReference() {}; + return apiClient.invokeAPI( + localVarPath, + "POST", + localVarQueryParams, + localVarCollectionQueryParams, + localVarQueryStringJoiner.toString(), + localVarPostBody, + localVarHeaderParams, + localVarCookieParams, + localVarFormParams, + localVarAccept, + localVarContentType, + localVarAuthNames, + localVarReturnType + ); + } + + @Override + public T invokeAPI(String url, String method, Object request, TypeReference returnType, Map additionalHeaders) throws ApiException { + String localVarPath = url.replace(apiClient.getBaseURL(), ""); + StringJoiner localVarQueryStringJoiner = new StringJoiner("&"); + List localVarQueryParams = new ArrayList(); + List localVarCollectionQueryParams = new ArrayList(); + Map localVarHeaderParams = new HashMap(); + Map localVarCookieParams = new HashMap(); + Map localVarFormParams = new HashMap(); + + localVarHeaderParams.putAll(additionalHeaders); + + final String[] localVarAccepts = { + "application/json" + }; + final String localVarAccept = apiClient.selectHeaderAccept(localVarAccepts); + + final String[] localVarContentTypes = { + "application/json" + }; + final String localVarContentType = apiClient.selectHeaderContentType(localVarContentTypes); + + String[] localVarAuthNames = new String[] { "basic" }; + + return apiClient.invokeAPI( + localVarPath, + method, + localVarQueryParams, + localVarCollectionQueryParams, + localVarQueryStringJoiner.toString(), + request, + localVarHeaderParams, + localVarCookieParams, + localVarFormParams, + localVarAccept, + localVarContentType, + localVarAuthNames, + returnType + ); + } +} diff --git a/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/model/LoggerInfo.java b/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/model/LoggerInfo.java new file mode 100644 index 00000000000..c3c1b1378f4 --- /dev/null +++ b/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/model/LoggerInfo.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.celeborn.rest.v1.model; + +import java.util.Objects; +import java.util.Arrays; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonTypeName; + +/** + * LoggerInfo + */ +@JsonPropertyOrder({ + LoggerInfo.JSON_PROPERTY_NAME, + LoggerInfo.JSON_PROPERTY_LEVEL +}) +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", comments = "Generator version: 7.8.0") +public class LoggerInfo { + public static final String JSON_PROPERTY_NAME = "name"; + private String name; + + public static final String JSON_PROPERTY_LEVEL = "level"; + private String level; + + public LoggerInfo() { + } + + public LoggerInfo name(String name) { + + this.name = name; + return this; + } + + /** + * The logger name. + * @return name + */ + @javax.annotation.Nonnull + @JsonProperty(JSON_PROPERTY_NAME) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + + public String getName() { + return name; + } + + + @JsonProperty(JSON_PROPERTY_NAME) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public void setName(String name) { + this.name = name; + } + + public LoggerInfo level(String level) { + + this.level = level; + return this; + } + + /** + * The logger level. + * @return level + */ + @javax.annotation.Nonnull + @JsonProperty(JSON_PROPERTY_LEVEL) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + + public String getLevel() { + return level; + } + + + @JsonProperty(JSON_PROPERTY_LEVEL) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public void setLevel(String level) { + this.level = level; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LoggerInfo loggerInfo = (LoggerInfo) o; + return Objects.equals(this.name, loggerInfo.name) && + Objects.equals(this.level, loggerInfo.level); + } + + @Override + public int hashCode() { + return Objects.hash(name, level); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class LoggerInfo {\n"); + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" level: ").append(toIndentedString(level)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + +} + diff --git a/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/model/LoggerInfos.java b/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/model/LoggerInfos.java new file mode 100644 index 00000000000..17e153db9e6 --- /dev/null +++ b/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/model/LoggerInfos.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.celeborn.rest.v1.model; + +import java.util.Objects; +import java.util.Arrays; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.annotation.JsonValue; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.celeborn.rest.v1.model.LoggerInfo; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonTypeName; + +/** + * LoggerInfos + */ +@JsonPropertyOrder({ + LoggerInfos.JSON_PROPERTY_LOGGERS +}) +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", comments = "Generator version: 7.8.0") +public class LoggerInfos { + public static final String JSON_PROPERTY_LOGGERS = "loggers"; + private List loggers = new ArrayList<>(); + + public LoggerInfos() { + } + + public LoggerInfos loggers(List loggers) { + + this.loggers = loggers; + return this; + } + + public LoggerInfos addLoggersItem(LoggerInfo loggersItem) { + if (this.loggers == null) { + this.loggers = new ArrayList<>(); + } + this.loggers.add(loggersItem); + return this; + } + + /** + * The logger infos. + * @return loggers + */ + @javax.annotation.Nullable + @JsonProperty(JSON_PROPERTY_LOGGERS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + + public List getLoggers() { + return loggers; + } + + + @JsonProperty(JSON_PROPERTY_LOGGERS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setLoggers(List loggers) { + this.loggers = loggers; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LoggerInfos loggerInfos = (LoggerInfos) o; + return Objects.equals(this.loggers, loggerInfos.loggers); + } + + @Override + public int hashCode() { + return Objects.hash(loggers); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class LoggerInfos {\n"); + sb.append(" loggers: ").append(toIndentedString(loggers)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + +} + diff --git a/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/worker/LoggerApi.java b/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/worker/LoggerApi.java new file mode 100644 index 00000000000..54fdcca0dac --- /dev/null +++ b/openapi/openapi-client/src/main/java/org/apache/celeborn/rest/v1/worker/LoggerApi.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.celeborn.rest.v1.worker; + +import com.fasterxml.jackson.core.type.TypeReference; + +import org.apache.celeborn.rest.v1.worker.invoker.ApiException; +import org.apache.celeborn.rest.v1.worker.invoker.ApiClient; +import org.apache.celeborn.rest.v1.worker.invoker.BaseApi; +import org.apache.celeborn.rest.v1.worker.invoker.Configuration; +import org.apache.celeborn.rest.v1.worker.invoker.Pair; + +import org.apache.celeborn.rest.v1.model.HandleResponse; +import org.apache.celeborn.rest.v1.model.LoggerInfo; +import org.apache.celeborn.rest.v1.model.LoggerInfos; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; + +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", comments = "Generator version: 7.8.0") +public class LoggerApi extends BaseApi { + + public LoggerApi() { + super(Configuration.getDefaultApiClient()); + } + + public LoggerApi(ApiClient apiClient) { + super(apiClient); + } + + /** + * + * Get the logger level, return all loggers if no name specified. + * @param name The logger name. (optional) + * @param all Return all logger instances if true, otherwise return all configured loggers. (optional, default to false) + * @return LoggerInfos + * @throws ApiException if fails to make API call + */ + public LoggerInfos getLogger(String name, Boolean all) throws ApiException { + return this.getLogger(name, all, Collections.emptyMap()); + } + + + /** + * + * Get the logger level, return all loggers if no name specified. + * @param name The logger name. (optional) + * @param all Return all logger instances if true, otherwise return all configured loggers. (optional, default to false) + * @param additionalHeaders additionalHeaders for this call + * @return LoggerInfos + * @throws ApiException if fails to make API call + */ + public LoggerInfos getLogger(String name, Boolean all, Map additionalHeaders) throws ApiException { + Object localVarPostBody = null; + + // create path and map variables + String localVarPath = "/api/v1/loggers"; + + StringJoiner localVarQueryStringJoiner = new StringJoiner("&"); + String localVarQueryParameterBaseName; + List localVarQueryParams = new ArrayList(); + List localVarCollectionQueryParams = new ArrayList(); + Map localVarHeaderParams = new HashMap(); + Map localVarCookieParams = new HashMap(); + Map localVarFormParams = new HashMap(); + + localVarQueryParams.addAll(apiClient.parameterToPair("name", name)); + localVarQueryParams.addAll(apiClient.parameterToPair("all", all)); + + localVarHeaderParams.putAll(additionalHeaders); + + + + final String[] localVarAccepts = { + "application/json" + }; + final String localVarAccept = apiClient.selectHeaderAccept(localVarAccepts); + + final String[] localVarContentTypes = { + + }; + final String localVarContentType = apiClient.selectHeaderContentType(localVarContentTypes); + + String[] localVarAuthNames = new String[] { "basic" }; + + TypeReference localVarReturnType = new TypeReference() {}; + return apiClient.invokeAPI( + localVarPath, + "GET", + localVarQueryParams, + localVarCollectionQueryParams, + localVarQueryStringJoiner.toString(), + localVarPostBody, + localVarHeaderParams, + localVarCookieParams, + localVarFormParams, + localVarAccept, + localVarContentType, + localVarAuthNames, + localVarReturnType + ); + } + + /** + * + * Set the logger level. + * @param loggerInfo (optional) + * @return HandleResponse + * @throws ApiException if fails to make API call + */ + public HandleResponse setLogger(LoggerInfo loggerInfo) throws ApiException { + return this.setLogger(loggerInfo, Collections.emptyMap()); + } + + + /** + * + * Set the logger level. + * @param loggerInfo (optional) + * @param additionalHeaders additionalHeaders for this call + * @return HandleResponse + * @throws ApiException if fails to make API call + */ + public HandleResponse setLogger(LoggerInfo loggerInfo, Map additionalHeaders) throws ApiException { + Object localVarPostBody = loggerInfo; + + // create path and map variables + String localVarPath = "/api/v1/loggers"; + + StringJoiner localVarQueryStringJoiner = new StringJoiner("&"); + String localVarQueryParameterBaseName; + List localVarQueryParams = new ArrayList(); + List localVarCollectionQueryParams = new ArrayList(); + Map localVarHeaderParams = new HashMap(); + Map localVarCookieParams = new HashMap(); + Map localVarFormParams = new HashMap(); + + + localVarHeaderParams.putAll(additionalHeaders); + + + + final String[] localVarAccepts = { + "application/json" + }; + final String localVarAccept = apiClient.selectHeaderAccept(localVarAccepts); + + final String[] localVarContentTypes = { + "application/json" + }; + final String localVarContentType = apiClient.selectHeaderContentType(localVarContentTypes); + + String[] localVarAuthNames = new String[] { "basic" }; + + TypeReference localVarReturnType = new TypeReference() {}; + return apiClient.invokeAPI( + localVarPath, + "POST", + localVarQueryParams, + localVarCollectionQueryParams, + localVarQueryStringJoiner.toString(), + localVarPostBody, + localVarHeaderParams, + localVarCookieParams, + localVarFormParams, + localVarAccept, + localVarContentType, + localVarAuthNames, + localVarReturnType + ); + } + + @Override + public T invokeAPI(String url, String method, Object request, TypeReference returnType, Map additionalHeaders) throws ApiException { + String localVarPath = url.replace(apiClient.getBaseURL(), ""); + StringJoiner localVarQueryStringJoiner = new StringJoiner("&"); + List localVarQueryParams = new ArrayList(); + List localVarCollectionQueryParams = new ArrayList(); + Map localVarHeaderParams = new HashMap(); + Map localVarCookieParams = new HashMap(); + Map localVarFormParams = new HashMap(); + + localVarHeaderParams.putAll(additionalHeaders); + + final String[] localVarAccepts = { + "application/json" + }; + final String localVarAccept = apiClient.selectHeaderAccept(localVarAccepts); + + final String[] localVarContentTypes = { + "application/json" + }; + final String localVarContentType = apiClient.selectHeaderContentType(localVarContentTypes); + + String[] localVarAuthNames = new String[] { "basic" }; + + return apiClient.invokeAPI( + localVarPath, + method, + localVarQueryParams, + localVarCollectionQueryParams, + localVarQueryStringJoiner.toString(), + request, + localVarHeaderParams, + localVarCookieParams, + localVarFormParams, + localVarAccept, + localVarContentType, + localVarAuthNames, + returnType + ); + } +} diff --git a/openapi/openapi-client/src/main/openapi3/master_rest_v1.yaml b/openapi/openapi-client/src/main/openapi3/master_rest_v1.yaml index 6ba0b40b345..c135eb7d77f 100644 --- a/openapi/openapi-client/src/main/openapi3/master_rest_v1.yaml +++ b/openapi/openapi-client/src/main/openapi3/master_rest_v1.yaml @@ -456,6 +456,51 @@ paths: type: string format: binary + /api/v1/loggers: + get: + tags: + - Logger + operationId: getLogger + description: Get the logger level, return all loggers if no name specified. + parameters: + - name: name + in: query + description: The logger name. + required: false + schema: + type: string + - name: all + in: query + description: Return all logger instances if true, otherwise return all configured loggers. + required: false + schema: + type: boolean + default: false + responses: + "200": + description: The request was successful. + content: + application/json: + schema: + $ref: '#/components/schemas/LoggerInfos' + post: + tags: + - Logger + operationId: setLogger + description: Set the logger level. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/LoggerInfo' + responses: + "200": + description: The request was successful. + content: + application/json: + schema: + $ref: '#/components/schemas/HandleResponse' + components: schemas: ConfigData: @@ -1067,6 +1112,28 @@ components: type: integer format: int32 + LoggerInfo: + type: object + properties: + name: + type: string + description: The logger name. + level: + type: string + description: The logger level. + required: + - name + - level + + LoggerInfos: + type: object + properties: + loggers: + type: array + description: The logger infos. + items: + $ref: '#/components/schemas/LoggerInfo' + HandleResponse: type: object properties: diff --git a/openapi/openapi-client/src/main/openapi3/worker_rest_v1.yaml b/openapi/openapi-client/src/main/openapi3/worker_rest_v1.yaml index 76875fa60b6..830d7721144 100644 --- a/openapi/openapi-client/src/main/openapi3/worker_rest_v1.yaml +++ b/openapi/openapi-client/src/main/openapi3/worker_rest_v1.yaml @@ -202,6 +202,51 @@ paths: schema: $ref: '#/components/schemas/ApplicationsResponse' + /api/v1/loggers: + get: + tags: + - Logger + operationId: getLogger + description: Get the logger level, return all loggers if no name specified. + parameters: + - name: name + in: query + description: The logger name. + required: false + schema: + type: string + - name: all + in: query + description: Return all logger instances if true, otherwise return all configured loggers. + required: false + schema: + type: boolean + default: false + responses: + "200": + description: The request was successful. + content: + application/json: + schema: + $ref: '#/components/schemas/LoggerInfos' + post: + tags: + - Logger + operationId: setLogger + description: Set the logger level. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/LoggerInfo' + responses: + "200": + description: The request was successful. + content: + application/json: + schema: + $ref: '#/components/schemas/HandleResponse' + components: schemas: ConfigData: @@ -651,6 +696,28 @@ components: - IMMEDIATELY - NONE + LoggerInfo: + type: object + properties: + name: + type: string + description: The logger name. + level: + type: string + description: The logger level. + required: + - name + - level + + LoggerInfos: + type: object + properties: + loggers: + type: array + description: The logger infos. + items: + $ref: '#/components/schemas/LoggerInfo' + HandleResponse: type: object properties: diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index 1243897adf6..b51e29d061d 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -135,6 +135,8 @@ object Dependencies { "org.fusesource.leveldbjni" } val leveldbJniAll = leveldbJniGroup % "leveldbjni-all" % leveldbJniVersion + val log4jApi = "org.apache.logging.log4j" % "log4j-api" % log4j2Version + val log4jCore = "org.apache.logging.log4j" % "log4j-core" % log4j2Version val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion @@ -629,6 +631,8 @@ object CelebornService { Dependencies.jettyServer, Dependencies.jettyServlet, Dependencies.jettyProxy, + Dependencies.log4jApi, + Dependencies.log4jCore, Dependencies.log4jSlf4jImpl % "test", Dependencies.log4j12Api % "test", Dependencies.h2 % "test", diff --git a/service/pom.xml b/service/pom.xml index 075e1543754..8d97d20a80d 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -180,6 +180,16 @@ jackson-databind-nullable + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + org.mockito diff --git a/service/src/main/scala/org/apache/celeborn/server/common/http/api/v1/ApiV1BaseResource.scala b/service/src/main/scala/org/apache/celeborn/server/common/http/api/v1/ApiV1BaseResource.scala index 908a2667cd6..110aeaffb2c 100644 --- a/service/src/main/scala/org/apache/celeborn/server/common/http/api/v1/ApiV1BaseResource.scala +++ b/service/src/main/scala/org/apache/celeborn/server/common/http/api/v1/ApiV1BaseResource.scala @@ -35,6 +35,9 @@ class ApiV1BaseResource extends ApiRequestContext { @Path("conf") def conf: Class[ConfResource] = classOf[ConfResource] + @Path("loggers") + def logger: Class[LoggerResource] = classOf[LoggerResource] + @Path("/thread_dump") @ApiResponse( responseCode = "200", diff --git a/service/src/main/scala/org/apache/celeborn/server/common/http/api/v1/LoggerResource.scala b/service/src/main/scala/org/apache/celeborn/server/common/http/api/v1/LoggerResource.scala new file mode 100644 index 00000000000..641debbfed8 --- /dev/null +++ b/service/src/main/scala/org/apache/celeborn/server/common/http/api/v1/LoggerResource.scala @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.celeborn.server.common.http.api.v1 + +import javax.ws.rs.{Consumes, DefaultValue, GET, POST, Produces, QueryParam} +import javax.ws.rs.core.MediaType + +import scala.collection.JavaConverters._ + +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.media.{Content, Schema} +import io.swagger.v3.oas.annotations.responses.ApiResponse +import io.swagger.v3.oas.annotations.tags.Tag +import org.apache.logging.log4j.{Level, LogManager} +import org.apache.logging.log4j.core.LoggerContext +import org.apache.logging.log4j.core.config.Configurator + +import org.apache.celeborn.rest.v1.model.{HandleResponse, LoggerInfo, LoggerInfos} +import org.apache.celeborn.server.common.http.api.ApiRequestContext + +@Tag(name = "Logger") +@Produces(Array(MediaType.APPLICATION_JSON)) +@Consumes(Array(MediaType.APPLICATION_JSON)) +class LoggerResource extends ApiRequestContext { + + @ApiResponse( + responseCode = "200", + content = Array(new Content( + mediaType = MediaType.APPLICATION_JSON, + schema = new Schema(implementation = classOf[LoggerInfo]))), + description = "Get the logger level, return all loggers if no name specified.") + @GET + def getLoggerLevel( + @QueryParam("name") name: String, + @QueryParam("all") @DefaultValue("false") @Parameter(description = + "Return all logger instances if true, otherwise return all configured loggers.") all: Boolean) + : LoggerInfos = { + if (null != name) { + new LoggerInfos().addLoggersItem( + new LoggerInfo().name(name).level(LogManager.getLogger(name).getLevel.toString)) + } else { + val loggerContext = LogManager.getContext(false).asInstanceOf[LoggerContext] + val loggers = + if (all) { + loggerContext.getLoggers.asScala.map { logger => + new LoggerInfo().name(logger.getName).level(logger.getLevel.toString) + }.toSeq + } else { + loggerContext.getConfiguration.getLoggers.values().asScala.map { loggerConfig => + new LoggerInfo().name(loggerConfig.getName).level(loggerConfig.getLevel.toString) + }.toSeq + } + new LoggerInfos().loggers(loggers.sortBy(_.getName).asJava) + } + } + + @ApiResponse( + responseCode = "200", + content = Array(new Content( + mediaType = MediaType.APPLICATION_JSON, + schema = new Schema(implementation = classOf[HandleResponse]))), + description = "Set the logger level.") + @POST + def setLoggerLevel(request: LoggerInfo): HandleResponse = { + val loggerName = request.getName + val logger = LogManager.getLogger(loggerName) + val originalLevel = logger.getLevel + val newLevel = Level.toLevel(request.getLevel) + Configurator.setLevel(loggerName, newLevel) + new HandleResponse().success(true).message( + s"Set logger `$loggerName` level from `$originalLevel` to `$newLevel`.`") + } +} diff --git a/service/src/test/scala/org/apache/celeborn/server/common/http/api/v1/ApiV1BaseResourceSuite.scala b/service/src/test/scala/org/apache/celeborn/server/common/http/api/v1/ApiV1BaseResourceSuite.scala index 876f3dbecfe..798eedb935d 100644 --- a/service/src/test/scala/org/apache/celeborn/server/common/http/api/v1/ApiV1BaseResourceSuite.scala +++ b/service/src/test/scala/org/apache/celeborn/server/common/http/api/v1/ApiV1BaseResourceSuite.scala @@ -19,11 +19,12 @@ package org.apache.celeborn.server.common.http.api.v1 import java.net.URI import javax.servlet.http.HttpServletResponse +import javax.ws.rs.client.Entity import javax.ws.rs.core.{MediaType, UriBuilder} import scala.collection.JavaConverters._ -import org.apache.celeborn.rest.v1.model.{ConfResponse, ThreadStackResponse} +import org.apache.celeborn.rest.v1.model.{ConfResponse, LoggerInfo, LoggerInfos, ThreadStackResponse} import org.apache.celeborn.server.common.http.HttpTestHelper abstract class ApiV1BaseResourceSuite extends HttpTestHelper { @@ -40,6 +41,85 @@ abstract class ApiV1BaseResourceSuite extends HttpTestHelper { assert(response.readEntity(classOf[String]).contains("Dynamic configuration is disabled.")) } + test("logger resource") { + val loggerName = this.getClass.getName + + // set logger level to INFO as initial state + val response = webTarget.path("loggers").request(MediaType.APPLICATION_JSON).post(Entity.entity( + new LoggerInfo().name(loggerName).level("INFO"), + MediaType.APPLICATION_JSON)) + assert(HttpServletResponse.SC_OK == response.getStatus) + + // check logger level is INFO + val response1 = webTarget.path("loggers") + .queryParam("name", loggerName) + .request(MediaType.APPLICATION_JSON).get() + assert(HttpServletResponse.SC_OK == response.getStatus) + val loggerInfo = response1.readEntity(classOf[LoggerInfos]).getLoggers.get(0) + assert(loggerName == loggerInfo.getName) + assert(loggerInfo.getLevel == "INFO") + assert(log.isInfoEnabled) + assert(!log.isDebugEnabled) + + // set logger level to DEBUG + val response2 = + webTarget.path("loggers").request(MediaType.APPLICATION_JSON).post(Entity.entity( + new LoggerInfo().name(loggerName).level("DEBUG"), + MediaType.APPLICATION_JSON)) + assert(HttpServletResponse.SC_OK == response2.getStatus) + + // check logger level is DEBUG + val response3 = webTarget.path("loggers") + .queryParam("name", loggerName) + .request(MediaType.APPLICATION_JSON).get() + assert(HttpServletResponse.SC_OK == response.getStatus) + val loggerInfo2 = response3.readEntity(classOf[LoggerInfos]).getLoggers.get(0) + assert(loggerName == loggerInfo2.getName) + assert(loggerInfo2.getLevel == "DEBUG") + assert(log.isInfoEnabled) + assert(log.isDebugEnabled) + + // check all configured loggers + val response4 = + webTarget.path("loggers").queryParam("all", "false").request(MediaType.APPLICATION_JSON).get() + assert(HttpServletResponse.SC_OK == response4.getStatus) + val configuredLoggers = response4.readEntity(classOf[LoggerInfos]).getLoggers.asScala + assert(configuredLoggers.exists(l => l.getName == loggerName && l.getLevel == "DEBUG")) + // root logger + assert(configuredLoggers.exists(l => l.getName == "" && l.getLevel == "INFO")) + + // check all loggers + val response5 = + webTarget.path("loggers").queryParam("all", "true").request(MediaType.APPLICATION_JSON).get() + assert(HttpServletResponse.SC_OK == response5.getStatus) + val allLoggers = response5.readEntity(classOf[LoggerInfos]).getLoggers.asScala + assert(configuredLoggers.exists(l => l.getName == loggerName && l.getLevel == "DEBUG")) + assert(allLoggers.size > configuredLoggers.size) + + // update root logger level + val response6 = + webTarget.path("loggers").request(MediaType.APPLICATION_JSON).post(Entity.entity( + new LoggerInfo().name("").level("DEBUG"), + MediaType.APPLICATION_JSON)) + assert(HttpServletResponse.SC_OK == response6.getStatus) + + // check root logger level is DEBUG + val response7 = webTarget.path("loggers") + .queryParam("name", "") + .request(MediaType.APPLICATION_JSON).get() + assert(HttpServletResponse.SC_OK == response7.getStatus) + val loggerInfo3 = response7.readEntity(classOf[LoggerInfos]).getLoggers.get(0) + assert("" == loggerInfo3.getName) + assert(loggerInfo3.getLevel == "DEBUG") + + // reset root logger level to INFO + val response8 = + webTarget.path("loggers").request(MediaType.APPLICATION_JSON).post(Entity.entity( + new LoggerInfo().name("").level("INFO"), + MediaType.APPLICATION_JSON)) + assert(HttpServletResponse.SC_OK == response8.getStatus) + } + test("thread_dump") { val response = webTarget.path("thread_dump").request(MediaType.APPLICATION_JSON).get() assert(HttpServletResponse.SC_OK == response.getStatus)