From 27e61994a14ce46ab4b71af29da1fb1dddb73b73 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sat, 28 Mar 2020 07:58:49 +0530 Subject: [PATCH 01/37] JWT Auth partial implelentations --- .../foxtrot/core/auth/FoxtrotRole.java | 25 +++ .../com/flipkart/foxtrot/core/auth/User.java | 43 ++++++ .../flipkart/foxtrot/core/auth/UserType.java | 9 ++ .../core/querystore/QueryExecutor.java | 6 +- foxtrot-server/pom.xml | 25 ++- .../foxtrot/server/auth/AuthConfig.java | 17 +++ .../foxtrot/server/auth/AuthStore.java | 61 ++++++++ .../foxtrot/server/auth/ESAuthStore.java | 142 ++++++++++++++++++ .../foxtrot/server/auth/JwtConfig.java | 33 ++++ .../foxtrot/server/auth/RoleAuthorizer.java | 33 ++++ .../flipkart/foxtrot/server/auth/Token.java | 30 ++++ .../server/auth/TokenAuthenticator.java | 65 ++++++++ .../foxtrot/server/auth/TokenType.java | 23 +++ .../foxtrot/server/auth/UserPrincipal.java | 24 +++ .../server/resources/AnalyticsResource.java | 14 +- .../foxtrot/server/resources/Auth.java | 129 ++++++++++++++++ .../foxtrot/server/utils/AuthUtils.java | 41 +++++ 17 files changed, 714 insertions(+), 6 deletions(-) create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/UserType.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/RoleAuthorizer.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/UserPrincipal.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java new file mode 100644 index 000000000..188f140a9 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java @@ -0,0 +1,25 @@ +package com.flipkart.foxtrot.core.auth; + +import lombok.Getter; + +/** + * + */ +@Getter +public enum FoxtrotRole { + INGEST(Value.INGEST), + CONSOLE(Value.CONSOLE), + QUERY(Value.QUERY); + + FoxtrotRole(String value) { + this.value = value; + } + + public static class Value { + public static final String INGEST = "INGEST"; + public static final String CONSOLE = "CONSOLE"; + public static final String QUERY = "QUERY"; + } + + private final String value; +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java new file mode 100644 index 000000000..a8648169d --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java @@ -0,0 +1,43 @@ +package com.flipkart.foxtrot.core.auth; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Value; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.validation.constraints.NotNull; +import java.util.Collections; +import java.util.Date; +import java.util.EnumSet; +import java.util.Set; + +/** + * + */ +@Value +public class User { + public static final User DEFAULT + = new User("__DEFAULT__", EnumSet.allOf(FoxtrotRole.class), Collections.emptySet(), new Date(), new Date()); + + @NotNull + @NotEmpty + String id; + @NotNull + @NotEmpty + Set roles; + Set tables; + Date created; + Date updated; + + public User( + @JsonProperty("id") String id, + @JsonProperty("roles") Set roles, + @JsonProperty("tables") Set tables, + @JsonProperty("created") Date created, + @JsonProperty("updated") Date updated) { + this.id = id; + this.roles = roles; + this.tables = tables; + this.created = created; + this.updated = updated; + } +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/UserType.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/UserType.java new file mode 100644 index 000000000..4457fedcb --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/UserType.java @@ -0,0 +1,9 @@ +package com.flipkart.foxtrot.core.auth; + +/** + * + */ +public enum UserType { + HUMAN, + SYSTEM +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryExecutor.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryExecutor.java index e4c66617b..989f29c20 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryExecutor.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryExecutor.java @@ -63,7 +63,7 @@ public ActionValidationResponse validate(T request) { public ActionResponse execute(T request) { Stopwatch stopwatch = Stopwatch.createStarted(); - Action action = null; + Action action = null; ActionEvaluationResponse evaluationResponse = null; try { action = resolve(request); @@ -108,8 +108,8 @@ public AsyncDataToken executeAsync(T request) { return dataToken; } - public Action resolve(T request) { - Action action; + public Action resolve(T request) { + Action action; action = analyticsLoader.getAction(request); if(null == action) { throw FoxtrotExceptions.createUnresolvableActionException(request); diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 1a19bf4fb..7cf2e59c7 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -11,12 +11,21 @@ foxtrot-server + + 1.2.2-0 + + io.dropwizard dropwizard-core ${dropwizard.version} + + io.dropwizard + dropwizard-auth + ${dropwizard.version} + ru.vyarus dropwizard-guicey @@ -48,7 +57,21 @@ - + + com.github.toastshaman + dropwizard-auth-jwt + ${dropwizard.jwt.auth.version} + + + io.dropwizard + dropwizard-bom + + + io.dropwizard + dropwizard-auth + + + com.flipkart.foxtrot diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java new file mode 100644 index 000000000..ac1b22761 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java @@ -0,0 +1,17 @@ +package com.flipkart.foxtrot.server.auth; + +import lombok.Data; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; + +/** + * + */ +@Data +public class AuthConfig { + private boolean disabled; + @NotNull + @Valid + private JwtConfig jwt = new JwtConfig(); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java new file mode 100644 index 000000000..7097e47e3 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java @@ -0,0 +1,61 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import lombok.val; + +import java.util.Date; +import java.util.HashSet; +import java.util.Optional; +import java.util.function.UnaryOperator; + +/** + * + */ +public interface AuthStore { + Optional provision(final User user); + + Optional getUser(final String userId); + + boolean deleteUser(final String id); + + boolean updateUser(final String id, UnaryOperator mutator); + + default boolean grantRole(final String userId, final FoxtrotRole role) { + return updateUser(userId, user -> { + val roles = user.getRoles() == null ? new HashSet() : user.getRoles(); + roles.add(role); + return new User(userId, roles, user.getTables(), user.getCreated(), new Date()); + }); + } + + default boolean revokeRole(final String userId, final FoxtrotRole role) { + return updateUser(userId, user -> { + val roles = user.getRoles() == null ? new HashSet() : user.getRoles(); + roles.remove(role); + return new User(userId, roles, user.getTables(), user.getCreated(), new Date()); + }); + } + + default boolean grantTableAccess(final String userId, final String table) { + return updateUser(userId, user -> { + val tables = user.getTables() == null ? new HashSet() : user.getTables(); + tables.add(table); + return new User(userId, user.getRoles(), tables, user.getCreated(), new Date()); + }); + } + + default boolean revokeTableAccess(final String userId, final String table) { + return updateUser(userId, user -> { + val tables = user.getTables() == null ? new HashSet() : user.getTables(); + tables.remove(table); + return new User(userId, user.getRoles(), tables, user.getCreated(), new Date()); + }); + } + + Optional provisionToken(final String userId, TokenType tokenType, Date expiry); + + Optional getToken(final String tokenId); + + boolean deleteToken(final String tokenId); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java new file mode 100644 index 000000000..1ccfca6d0 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -0,0 +1,142 @@ +package com.flipkart.foxtrot.server.auth; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import lombok.SneakyThrows; +import lombok.val; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.rest.RestStatus; + +import javax.inject.Inject; +import java.util.Date; +import java.util.Optional; +import java.util.UUID; +import java.util.function.UnaryOperator; + +/** + * + */ + +public class ESAuthStore implements AuthStore { + private static final String USERS_INDEX = "user-meta"; + private static final String TOKENS_INDEX = "tokens"; + + private final ElasticsearchConnection connection; + private final ObjectMapper mapper; + + @Inject + public ESAuthStore(ElasticsearchConnection connection, ObjectMapper mapper) { + this.connection = connection; + this.mapper = mapper; + } + + @Override + @SneakyThrows + public Optional provision(User user) { + val status = saveUser(user, DocWriteRequest.OpType.CREATE); + if (status != RestStatus.OK) { + return Optional.empty(); + } + return getUser(user.getId()); + } + + @Override + @SneakyThrows + public Optional getUser(String userId) { + val getResp = connection.getClient() + .get(new GetRequest(USERS_INDEX) + .id(userId)) + .actionGet(); + if (!getResp.isExists()) { + return Optional.empty(); + } + return Optional.of(mapper.readValue(getResp.getSourceAsString(), User.class)); + } + + @Override + public boolean deleteUser(String id) { + return connection.getClient() + .delete(new DeleteRequest(USERS_INDEX) + .id(id) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) + .actionGet() + .status() == RestStatus.OK; + } + + @Override + @SneakyThrows + public boolean updateUser( + String id, UnaryOperator mutator) { + val user = getUser(id).orElse(null); + if (null == user) { + return false; + } + final User updatedUser = mutator.apply(user); + return saveUser(updatedUser, DocWriteRequest.OpType.UPDATE) == RestStatus.OK; + } + + @Override + @SneakyThrows + public Optional provisionToken(String userId, TokenType tokenType, Date expiry) { + val userPresent = getUser(userId).isPresent(); + if (!userPresent) { + return Optional.empty(); + } + final String tokenId = UUID.randomUUID().toString(); + val saveStatus = connection.getClient() + .index(new IndexRequest(TOKENS_INDEX) + .source(mapper.writeValueAsString(new Token(tokenId, tokenType, userId, expiry))) + .id(userId) + .type("TOKEN") + .create(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) + .actionGet() + .status(); + if (saveStatus != RestStatus.OK) { + return Optional.empty(); + } + return getToken(tokenId); + } + + @Override + @SneakyThrows + public Optional getToken(String tokenId) { + val getResp = connection.getClient() + .get(new GetRequest(TOKENS_INDEX) + .id(tokenId)) + .actionGet(); + if (!getResp.isExists()) { + return Optional.empty(); + } + return Optional.of(mapper.readValue(getResp.getSourceAsString(), Token.class)); + } + + @Override + public boolean deleteToken(String tokenId) { + return connection.getClient() + .delete(new DeleteRequest(TOKENS_INDEX) + .id(tokenId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) + .actionGet() + .status() == RestStatus.OK; + } + + + private RestStatus saveUser(User user, DocWriteRequest.OpType opType) throws JsonProcessingException { + return connection.getClient() + .index(new IndexRequest(USERS_INDEX) + .source(mapper.writeValueAsString(user)) + .id(user.getId()) + .type("USER") + .opType(opType) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) + .actionGet() + .status(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java new file mode 100644 index 000000000..3c13ece5e --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java @@ -0,0 +1,33 @@ +package com.flipkart.foxtrot.server.auth; + +import com.google.common.annotations.VisibleForTesting; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.validation.constraints.NotNull; + +/** + * + */ +@Data +@NoArgsConstructor +public class JwtConfig { + @NotNull + @NotEmpty + private String privateKey; + + @NotNull + @NotEmpty + private String issuerId; + + @NotEmpty + @NotNull + private String authCachePolicy = "maximumSize=10000, expireAfterAccess=10m"; + + @VisibleForTesting + public JwtConfig(String privateKey, String issuerId) { + this.privateKey = privateKey; + this.issuerId = issuerId; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/RoleAuthorizer.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/RoleAuthorizer.java new file mode 100644 index 000000000..82a2c5fb7 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/RoleAuthorizer.java @@ -0,0 +1,33 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import io.dropwizard.auth.Authorizer; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import javax.inject.Singleton; + +/** + * + */ +@Singleton +@Slf4j +public class RoleAuthorizer implements Authorizer { + @Override + public boolean authorize(UserPrincipal userPrincipal, String role) { + val user = userPrincipal.getUser(); + val foxtrotRole = FoxtrotRole.valueOf(role); + + if(!user.getRoles().contains(foxtrotRole)) { + log.warn("User {} is trying to access unauthorized role: {}", user.getId(), role); + return false; + } + val token = userPrincipal.getToken(); + if(!token.getTokenType().getAllowedRoles().contains(foxtrotRole)) { + log.warn("User {} trying to access resource with role: {} with token of tye: {}", + user.getId(), role, token.getTokenType().name()); + return false; + } + return true; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java new file mode 100644 index 000000000..2e00df3ce --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java @@ -0,0 +1,30 @@ +package com.flipkart.foxtrot.server.auth; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Value; + +import java.util.Date; + +/** + * + */ +@Value +public class Token { + public static final Token DEFAULT = new Token("__DEFAULT_TOKEN__", TokenType.SYSTEM, "__DEFAULT__", null); + + String id; + TokenType tokenType; + String userId; + Date expiry; + + public Token( + @JsonProperty("id") String id, + @JsonProperty("tokenType") TokenType tokenType, + @JsonProperty("userId") String userId, + @JsonProperty("expiry") Date expiry) { + this.id = id; + this.tokenType = tokenType; + this.userId = userId; + this.expiry = expiry; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java new file mode 100644 index 000000000..22521ad68 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java @@ -0,0 +1,65 @@ +package com.flipkart.foxtrot.server.auth; + +import io.dropwizard.auth.AuthenticationException; +import io.dropwizard.auth.Authenticator; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.jose4j.jwt.MalformedClaimException; +import org.jose4j.jwt.consumer.JwtContext; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.Optional; + +/** + * Authenticator that will be run + */ +@Slf4j +@Singleton +public class TokenAuthenticator implements Authenticator { + + private final AuthConfig config; + private final Provider authStore; + @Inject + public TokenAuthenticator(AuthConfig config, final Provider authStore) { + this.config = config; + this.authStore = authStore; + } + + @Override + public Optional authenticate(JwtContext jwtContext) throws AuthenticationException { + if(config.isDisabled()) { + log.debug("Authentication is disabled"); + return Optional.of(UserPrincipal.DEFAULT); + } + final String userId; + final String tokenId; + try { + val claims = jwtContext.getJwtClaims(); + userId = claims.getSubject(); + tokenId = claims.getJwtId(); + } + catch (MalformedClaimException e) { + log.error(String.format("exception in claim extraction %s", e.getMessage()), e); + return Optional.empty(); + } + log.debug("authentication_requested userId:{} tokenId:{}", userId, tokenId); + val token = authStore.get().getToken(tokenId).orElse(null); + if (token == null) { + log.warn("authentication_failed::invalid_session userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + if (!token.getUserId().equals(userId)) { + log.warn("authentication_failed::user_mismatch userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + val user = authStore.get().getUser(token.getUserId()).orElse(null); + if (null == user) { + log.warn("authentication_failed::invalid_user userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + log.debug("authentication_success userId:{} tokenId:{}", userId, tokenId); + return Optional.of(new UserPrincipal(user, token)); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java new file mode 100644 index 000000000..27a334c15 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java @@ -0,0 +1,23 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import lombok.Getter; + +import java.util.EnumSet; +import java.util.Set; + +/** + * + */ +@Getter +public enum TokenType { + DYNAMIC(EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.CONSOLE)), + STATIC(EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.INGEST)), + SYSTEM(EnumSet.allOf(FoxtrotRole.class)); + + private final Set allowedRoles; + + TokenType(Set allowedRoles) { + this.allowedRoles = allowedRoles; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/UserPrincipal.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/UserPrincipal.java new file mode 100644 index 000000000..b38e0ee0c --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/UserPrincipal.java @@ -0,0 +1,24 @@ +package com.flipkart.foxtrot.server.auth; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.flipkart.foxtrot.core.auth.User; +import lombok.Value; + +import java.security.Principal; + +/** + * + */ +@Value +public class UserPrincipal implements Principal { + public static final UserPrincipal DEFAULT = new UserPrincipal(User.DEFAULT, Token.DEFAULT); + + User user; + Token token; + + @Override + @JsonIgnore + public String getName() { + return user.getId(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java index 4989ae6ed..43f973ff7 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java @@ -19,11 +19,15 @@ import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionResponse; import com.flipkart.foxtrot.common.ActionValidationResponse; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.common.AsyncDataToken; import com.flipkart.foxtrot.core.querystore.QueryExecutor; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import io.dropwizard.auth.Auth; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.Valid; @@ -55,7 +59,10 @@ public AnalyticsResource(QueryExecutor queryExecutor) { @POST @Timed @ApiOperation("runSync") - public ActionResponse runSync(@Valid final ActionRequest request) { + @RolesAllowed(FoxtrotRole.Value.QUERY) + public ActionResponse runSync( + @Auth final UserPrincipal userPrincipal, + @Valid final ActionRequest request) { return queryExecutor.execute(request); } @@ -63,7 +70,10 @@ public ActionResponse runSync(@Valid final ActionRequest request) { @Path("/async") @Timed @ApiOperation("runSyncAsync") - public AsyncDataToken runSyncAsync(@Valid final ActionRequest request) { + @RolesAllowed(FoxtrotRole.Value.QUERY) + public AsyncDataToken runSyncAsync( + @Auth final UserPrincipal userPrincipal, + @Valid final ActionRequest request) { return queryExecutor.executeAsync(request); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java new file mode 100644 index 000000000..3bea02a1b --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java @@ -0,0 +1,129 @@ +package com.flipkart.foxtrot.server.resources; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.flipkart.foxtrot.server.auth.TokenType; +import com.flipkart.foxtrot.server.utils.AuthUtils; +import io.swagger.annotations.Api; +import lombok.val; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Collections; + +/** + * + */ +@Path("/v1/auth") +@Produces(MediaType.APPLICATION_JSON) +@Api("Auth related APIs. DO NOT expose these for public access.") +public class Auth { + private final Provider authProvider; + private final AuthConfig authConfig; + + @Inject + public Auth(Provider authProvider, AuthConfig authConfig) { + this.authProvider = authProvider; + this.authConfig = authConfig; + } + + @POST + @Path("/users") + @Consumes(MediaType.APPLICATION_JSON) + public Response provisionUser(@NotNull @Valid final User user) { + return Response.ok(authProvider.get().provision(user)).build(); + } + + @GET + @Path("/users/{userId}") + public Response getUser(@NotNull @NotEmpty @PathParam("userId") final String userId) { + return Response.ok(authProvider.get().getUser(userId)).build(); + } + + @PUT + @Path("/users/{userId}/roles/grant/{role}") + public Response grantRole(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @PathParam("role") final FoxtrotRole role) { + val status = authProvider.get() + .grantRole(userId, role); + return updateUserResponse(userId, status); + } + + @PUT + @Path("/users/{userId}/roles/revoke/{role}") + public Response revokeRole(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @PathParam("role") final FoxtrotRole role) { + val status = authProvider.get() + .revokeRole(userId, role); + return updateUserResponse(userId, status); + } + + @PUT + @Path("/users/{userId}/tables/access/grant/{table}") + public Response grantTableAccess(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @NotEmpty @PathParam("table") final String table) { + val status = authProvider.get() + .grantTableAccess(userId, table); + return updateUserResponse(userId, status); + } + + @PUT + @Path("/users/{userId}/tables/access/revoke/{table}") + public Response revokeTableAccess(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @NotEmpty @PathParam("table") final String table) { + val status = authProvider.get() + .revokeTableAccess(userId, table); + return updateUserResponse(userId, status); + } + + @DELETE + @Path("/users/{userId}") + public Response deleteUser(@NotNull @NotEmpty @PathParam("userId") final String userId) { + final boolean status = authProvider.get().deleteUser(userId); + if(!status) { + return Response.notModified().build(); + } + return Response.ok().build(); + } + + @POST + @Path("/tokens/{userId}") + public Response provisionToken(@NotNull @NotEmpty @PathParam("userId") final String userId) { + val token = authProvider.get().provisionToken(userId, TokenType.STATIC, null).orElse(null); + if(null == token) { + return Response.notModified().build(); + } + return Response + .ok(Collections.singletonMap("jwt", AuthUtils.createJWT(token, authConfig.getJwt()))) + .build(); + } + + @DELETE + @Path("/tokens/{userId}") + public Response deleteToken(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @NotEmpty @PathParam("tokenId") final String tokenId) { + val status = authProvider.get().deleteToken(tokenId); + if(!status) { + return Response.notModified().build(); + } + return Response.ok().build(); + } + + private Response updateUserResponse(String userId, boolean status) { + if (!status) { + return Response.notModified() + .build(); + } + return Response.ok() + .entity(authProvider.get().getUser(userId)) + .build(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java new file mode 100644 index 000000000..bcc94282e --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java @@ -0,0 +1,41 @@ +package com.flipkart.foxtrot.server.utils; + +import com.flipkart.foxtrot.server.auth.JwtConfig; +import com.flipkart.foxtrot.server.auth.Token; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.SneakyThrows; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jws.JsonWebSignature; +import org.jose4j.jwt.JwtClaims; +import org.jose4j.jwt.NumericDate; +import org.jose4j.keys.HmacKey; + +import java.nio.charset.StandardCharsets; + +/** + * + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class AuthUtils { + @SneakyThrows + public static String createJWT(final Token token, final JwtConfig jwtConfig) { + JwtClaims claims = new JwtClaims(); + claims.setIssuer(jwtConfig.getIssuerId()); + claims.setGeneratedJwtId(); + claims.setIssuedAtToNow(); + claims.setJwtId(token.getId()); + claims.setNotBeforeMinutesInThePast(2); + claims.setSubject(token.getUserId()); + claims.setAudience(token.getTokenType().name()); + if(null != token.getExpiry()) { + claims.setExpirationTime(NumericDate.fromMilliseconds(token.getExpiry().getTime())); + } + JsonWebSignature jws = new JsonWebSignature(); + jws.setPayload(claims.toJson()); + final byte[] secretKey = jwtConfig.getPrivateKey().getBytes(StandardCharsets.UTF_8); + jws.setKey(new HmacKey(secretKey)); + jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.HMAC_SHA512); + return jws.getCompactSerialization(); + } +} From 84999af5e04bb6254d3dcbfac1bcabfbc857ecf0 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sun, 29 Mar 2020 08:30:41 +0530 Subject: [PATCH 02/37] Added auth to resources --- config/docker.yml | 5 ++ .../server/auth/JwtAuthDynamicFeature.java | 37 ++++++++++++ .../config/FoxtrotServerConfiguration.java | 5 ++ .../foxtrot/server/di/FoxtrotModule.java | 58 +++++++++++++++++-- .../server/resources/ConsoleResource.java | 4 ++ .../server/resources/DocumentResource.java | 6 ++ .../foxtrot/server/resources/FqlResource.java | 3 + 7 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java diff --git a/config/docker.yml b/config/docker.yml index ed65803fe..e31c03eb2 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -57,3 +57,8 @@ consoleHistoryConfig: interval: 86400 initialDelay: 10 +auth: + disabled: false + jwt: + issuerId: foxtrot-server + privateKey: "6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java new file mode 100644 index 000000000..f652e7c82 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java @@ -0,0 +1,37 @@ +package com.flipkart.foxtrot.server.auth; + +import com.github.toastshaman.dropwizard.auth.jwt.JwtAuthFilter; +import io.dropwizard.auth.*; +import io.dropwizard.setup.Environment; +import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature; +import org.jose4j.jwt.consumer.JwtConsumer; +import org.jose4j.jwt.consumer.JwtContext; + +import javax.inject.Inject; +import javax.ws.rs.ext.Provider; + +/** + * + */ +@Provider +public class JwtAuthDynamicFeature extends AuthDynamicFeature { + @Inject + public JwtAuthDynamicFeature( + JwtConsumer jwtConsumer, + Authorizer authorizer, + Authenticator authenticator, + Environment environment) { + super(new JwtAuthFilter.Builder() + .setJwtConsumer(jwtConsumer) + .setPrefix("Bearer") + .setRealm("realm") + .setAuthenticator(authenticator) + .setAuthorizer(authorizer) + .setUnauthorizedHandler(new DefaultUnauthorizedHandler()) + .buildAuthFilter()); + if(null != environment) { + environment.jersey().register(new AuthValueFactoryProvider.Binder<>(UserPrincipal.class)); + environment.jersey().register(RolesAllowedDynamicFeature.class); + } + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java index a2172dca0..b0dae4c10 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java @@ -25,6 +25,7 @@ import com.flipkart.foxtrot.core.querystore.impl.CacheConfig; import com.flipkart.foxtrot.core.querystore.impl.ClusterConfig; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConfig; +import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.server.jobs.consolehistory.ConsoleHistoryConfig; import com.foxtrot.flipkart.translator.config.SegregationConfiguration; import io.dropwizard.Configuration; @@ -79,6 +80,10 @@ public class FoxtrotServerConfiguration extends Configuration { @Builder.Default private TextNodeRemoverConfiguration textNodeRemover = new TextNodeRemoverConfiguration(); + @Valid + @NotNull + private AuthConfig auth; + public FoxtrotServerConfiguration() { this.hbase = new HbaseConfig(); this.elasticsearch = new ElasticsearchConfig(); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java index 14782446b..f56882430 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java @@ -34,6 +34,7 @@ import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.table.impl.DistributedTableMetadataManager; import com.flipkart.foxtrot.core.table.impl.FoxtrotTableManager; +import com.flipkart.foxtrot.server.auth.*; import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; import com.flipkart.foxtrot.server.console.ConsolePersistence; import com.flipkart.foxtrot.server.console.ElasticsearchConsolePersistence; @@ -41,22 +42,32 @@ import com.flipkart.foxtrot.sql.fqlstore.FqlStoreService; import com.flipkart.foxtrot.sql.fqlstore.FqlStoreServiceImpl; import com.foxtrot.flipkart.translator.config.SegregationConfiguration; +import com.google.common.cache.CacheBuilderSpec; import com.google.common.collect.ImmutableList; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.TypeLiteral; +import io.dropwizard.auth.Authenticator; +import io.dropwizard.auth.Authorizer; +import io.dropwizard.auth.CachingAuthenticator; import io.dropwizard.server.ServerFactory; import io.dropwizard.setup.Environment; import io.dropwizard.util.Duration; +import org.apache.hadoop.conf.Configuration; +import org.jose4j.jwa.AlgorithmConstraints; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jwt.consumer.JwtConsumer; +import org.jose4j.jwt.consumer.JwtConsumerBuilder; +import org.jose4j.jwt.consumer.JwtContext; +import org.jose4j.keys.HmacKey; -import java.io.IOException; -import java.util.Set; import javax.inject.Singleton; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; -import org.apache.hadoop.conf.Configuration; /** @@ -87,8 +98,11 @@ protected void configure() { .to(StrSubstitutorEmailBodyBuilder.class); bind(TableManager.class) .to(FoxtrotTableManager.class); - bind(new TypeLiteral>() { - }).toProvider(HealthcheckListProvider.class); + bind(new TypeLiteral>() {}).toProvider(HealthcheckListProvider.class); + bind(AuthStore.class) + .to(ESAuthStore.class); + bind(new TypeLiteral>() {}) + .to(RoleAuthorizer.class); } @Provides @@ -213,4 +227,38 @@ public Configuration provideHBaseConfiguration(HbaseConfig hbaseConfig) throws I public ServerFactory serverFactory(FoxtrotServerConfiguration configuration) { return configuration.getServerFactory(); } + + @Provides + @Singleton + public AuthConfig authConfig(FoxtrotServerConfiguration serverConfiguration) { + return serverConfiguration.getAuth(); + } + + @Provides + @com.google.inject.Singleton + public JwtConsumer provideJwtConsumer(AuthConfig config) { + final JwtConfig jwtConfig = config.getJwt(); + final byte[] secretKey = jwtConfig.getPrivateKey().getBytes(StandardCharsets.UTF_8); + return new JwtConsumerBuilder() + .setRequireIssuedAt() + .setRequireSubject() + .setExpectedIssuer(jwtConfig.getIssuerId()) + .setVerificationKey(new HmacKey(secretKey)) + .setJwsAlgorithmConstraints(new AlgorithmConstraints( + AlgorithmConstraints.ConstraintType.WHITELIST, + AlgorithmIdentifiers.HMAC_SHA512)) + .build(); + } + + @Provides + @Singleton + public Authenticator authenticator( + final Environment environment, + final TokenAuthenticator authenticator, + final AuthConfig authConfig) { + return new CachingAuthenticator<>( + environment.metrics(), + authenticator, + CacheBuilderSpec.parse(authConfig.getJwt().getAuthCachePolicy())); + } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java index 759edacb1..87f916647 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java @@ -15,11 +15,13 @@ */ package com.flipkart.foxtrot.server.resources; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.server.console.Console; import com.flipkart.foxtrot.server.console.ConsolePersistence; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.*; @@ -41,6 +43,7 @@ public ConsoleResource(ConsolePersistence consolePersistence) { } @POST + @RolesAllowed(FoxtrotRole.Value.CONSOLE) @ApiOperation("Save Console") public Console save(Console console) { consolePersistence.save(console); @@ -56,6 +59,7 @@ public Console get(@PathParam("id") final String id) { @DELETE @Path("/{id}/delete") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) @ApiOperation("Delete Console - via id") public void delete(@PathParam("id") final String id) { consolePersistence.delete(id); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java index 5f2a3b275..db94011e0 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java @@ -18,12 +18,14 @@ import com.codahale.metrics.annotation.Timed; import com.collections.CollectionUtils; import com.flipkart.foxtrot.common.Document; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.foxtrot.flipkart.translator.TableTranslator; import com.google.common.collect.Lists; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.Valid; @@ -59,6 +61,7 @@ public DocumentResource(QueryStore queryStore, TableTranslator tableTranslator) @POST @Consumes(MediaType.APPLICATION_JSON) @Timed + @RolesAllowed(FoxtrotRole.Value.INGEST) @ApiOperation("Save Document") public Response saveDocument(@PathParam("table") String table, @Valid final Document document) { String tableName = tableTranslator.getTable(table, document); @@ -76,6 +79,7 @@ public Response saveDocument(@PathParam("table") String table, @Valid final Docu @Path("/bulk") @Consumes(MediaType.APPLICATION_JSON) @Timed + @RolesAllowed(FoxtrotRole.Value.INGEST) @ApiOperation("Save list of documents") public Response saveDocuments(@PathParam("table") String table, @Valid final List documents) { Map> tableVsDocuments = getTableVsDocuments(table, documents); @@ -89,6 +93,7 @@ public Response saveDocuments(@PathParam("table") String table, @Valid final Lis @GET @Path("/{id}") @Timed + @RolesAllowed(FoxtrotRole.Value.QUERY) @ApiOperation("Get Document") public Response getDocument(@PathParam("table") final String table, @PathParam("id") @NotNull final String id) { return Response.ok(queryStore.get(table, id)) @@ -97,6 +102,7 @@ public Response getDocument(@PathParam("table") final String table, @PathParam(" @GET @Timed + @RolesAllowed(FoxtrotRole.Value.QUERY) @ApiOperation("Get Documents") public Response getDocuments(@PathParam("table") final String table, @QueryParam("id") @NotNull final List ids) { return Response.ok(queryStore.getAll(table, ids)) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/FqlResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/FqlResource.java index 8d1928e6a..4bddc322f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/FqlResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/FqlResource.java @@ -1,6 +1,7 @@ package com.flipkart.foxtrot.server.resources; import com.codahale.metrics.annotation.Timed; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.server.providers.FlatToCsvConverter; import com.flipkart.foxtrot.server.providers.FoxtrotExtraMediaType; import com.flipkart.foxtrot.sql.FqlEngine; @@ -12,6 +13,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.*; @@ -23,6 +25,7 @@ @Path("/v1/fql") @Api(value = "/v1/fql", description = "FQL API") @Singleton +@RolesAllowed(FoxtrotRole.Value.QUERY) public class FqlResource { private FqlEngine fqlEngine; private FqlStoreService fqlStoreService; From d2b31f7ff37428333389c0b7b95b8224bd30b85f Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sun, 29 Mar 2020 17:12:40 +0530 Subject: [PATCH 03/37] Auth store --- .../com/flipkart/foxtrot/core/auth/User.java | 20 ----------------- .../foxtrot/server/auth/ESAuthStore.java | 5 +++-- .../flipkart/foxtrot/server/auth/Token.java | 12 ---------- .../server/auth/io/CreateUserRequest.java | 22 +++++++++++++++++++ .../foxtrot/server/resources/Auth.java | 6 ++++- pom.xml | 1 + 6 files changed, 31 insertions(+), 35 deletions(-) create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java index a8648169d..b4291f5ab 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java @@ -1,10 +1,7 @@ package com.flipkart.foxtrot.core.auth; -import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Value; -import org.hibernate.validator.constraints.NotEmpty; -import javax.validation.constraints.NotNull; import java.util.Collections; import java.util.Date; import java.util.EnumSet; @@ -18,26 +15,9 @@ public class User { public static final User DEFAULT = new User("__DEFAULT__", EnumSet.allOf(FoxtrotRole.class), Collections.emptySet(), new Date(), new Date()); - @NotNull - @NotEmpty String id; - @NotNull - @NotEmpty Set roles; Set tables; Date created; Date updated; - - public User( - @JsonProperty("id") String id, - @JsonProperty("roles") Set roles, - @JsonProperty("tables") Set tables, - @JsonProperty("created") Date created, - @JsonProperty("updated") Date updated) { - this.id = id; - this.roles = roles; - this.tables = tables; - this.created = created; - this.updated = updated; - } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java index 1ccfca6d0..ec42a04d7 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import javax.inject.Inject; @@ -91,7 +92,7 @@ public Optional provisionToken(String userId, TokenType tokenType, Date e final String tokenId = UUID.randomUUID().toString(); val saveStatus = connection.getClient() .index(new IndexRequest(TOKENS_INDEX) - .source(mapper.writeValueAsString(new Token(tokenId, tokenType, userId, expiry))) + .source(mapper.writeValueAsString(new Token(tokenId, tokenType, userId, expiry)), XContentType.JSON) .id(userId) .type("TOKEN") .create(true) @@ -131,7 +132,7 @@ public boolean deleteToken(String tokenId) { private RestStatus saveUser(User user, DocWriteRequest.OpType opType) throws JsonProcessingException { return connection.getClient() .index(new IndexRequest(USERS_INDEX) - .source(mapper.writeValueAsString(user)) + .source(mapper.writeValueAsString(user), XContentType.JSON) .id(user.getId()) .type("USER") .opType(opType) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java index 2e00df3ce..1ffb56288 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java @@ -1,6 +1,5 @@ package com.flipkart.foxtrot.server.auth; -import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Value; import java.util.Date; @@ -16,15 +15,4 @@ public class Token { TokenType tokenType; String userId; Date expiry; - - public Token( - @JsonProperty("id") String id, - @JsonProperty("tokenType") TokenType tokenType, - @JsonProperty("userId") String userId, - @JsonProperty("expiry") Date expiry) { - this.id = id; - this.tokenType = tokenType; - this.userId = userId; - this.expiry = expiry; - } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java new file mode 100644 index 000000000..555664b82 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java @@ -0,0 +1,22 @@ +package com.flipkart.foxtrot.server.auth.io; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import lombok.Value; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.validation.constraints.NotNull; +import java.util.Set; + +/** + * + */ +@Value +public class CreateUserRequest { + @NotNull + @NotEmpty + String id; + @NotNull + @NotEmpty + Set roles; + Set tables; +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java index 3bea02a1b..9599673eb 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java @@ -5,6 +5,7 @@ import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.server.auth.AuthStore; import com.flipkart.foxtrot.server.auth.TokenType; +import com.flipkart.foxtrot.server.auth.io.CreateUserRequest; import com.flipkart.foxtrot.server.utils.AuthUtils; import io.swagger.annotations.Api; import lombok.val; @@ -18,6 +19,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.Collections; +import java.util.Date; /** * @@ -38,7 +40,9 @@ public Auth(Provider authProvider, AuthConfig authConfig) { @POST @Path("/users") @Consumes(MediaType.APPLICATION_JSON) - public Response provisionUser(@NotNull @Valid final User user) { + public Response provisionUser(@NotNull @Valid final CreateUserRequest createUserRequest) { + val user = new User(createUserRequest.getId(), createUserRequest.getRoles(), createUserRequest.getTables(), + new Date(), new Date()); return Response.ok(authProvider.get().provision(user)).build(); } diff --git a/pom.xml b/pom.xml index 10c773a4b..19461f9cc 100644 --- a/pom.xml +++ b/pom.xml @@ -292,6 +292,7 @@ 1.8 1.8 + -parameters From 6620022f08e755ef6bc1ba9f07b45baa9313bef4 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sun, 5 Apr 2020 18:15:48 +0530 Subject: [PATCH 04/37] Added api to get token --- config/docker.yml | 13 +++-- foxtrot-core/pom.xml | 4 +- .../foxtrot/server/InitializerCommand.java | 2 + .../foxtrot/server/auth/AuthStore.java | 4 ++ .../foxtrot/server/auth/ESAuthStore.java | 54 ++++++++++++++----- .../foxtrot/server/di/FoxtrotModule.java | 4 ++ .../foxtrot/server/resources/Auth.java | 7 +++ scripts/generate-events.py | 5 +- 8 files changed, 72 insertions(+), 21 deletions(-) diff --git a/config/docker.yml b/config/docker.yml index e31c03eb2..7f3b27ad8 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -6,6 +6,14 @@ server: - type: http port: 17001 +logging: + level: INFO + loggers: + com.flipkart.foxtrot: DEBUG + org.apache.zookeeper: WARN + org.apache.hadoop.hbase.zookeeper: WARN + org.apache.hadoop.hbase.client: WARN + elasticsearch: hosts: - ${ELASTICSEARCH_HOST} @@ -29,11 +37,6 @@ cacheConfig: maxIdleSeconds: 15 timeToLiveSeconds: 15 -logging: - level: INFO - loggers: - com.flipkart.foxtrot.core.querystore.actions: DEBUG - cardinality: enabled: true batchSize: 10 diff --git a/foxtrot-core/pom.xml b/foxtrot-core/pom.xml index 7fbb93a49..b760db56c 100644 --- a/foxtrot-core/pom.xml +++ b/foxtrot-core/pom.xml @@ -141,8 +141,8 @@ org.elasticsearch.client - transport - 6.0.1 + elasticsearch-rest-high-level-client + 6.3.1 diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java index 8bd50e128..e3bb8b284 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java @@ -63,6 +63,8 @@ protected void run(Bootstrap bootstrap, Namespace na createMetaIndex(connection, "consoles_v2", numReplicas); createMetaIndex(connection, "table-meta", numReplicas); createMetaIndex(connection, "consoles_history", numReplicas); + createMetaIndex(connection, "user-meta", numReplicas); + createMetaIndex(connection, "tokens", numReplicas); logger.info("Creating mapping"); PutIndexTemplateRequest putIndexTemplateRequest = ElasticsearchUtils.getClusterTemplateMapping(); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java index 7097e47e3..43387fcf0 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java @@ -2,6 +2,7 @@ import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.auth.User; +import lombok.SneakyThrows; import lombok.val; import java.util.Date; @@ -57,5 +58,8 @@ default boolean revokeTableAccess(final String userId, final String table) { Optional getToken(final String tokenId); + @SneakyThrows + Optional getTokenForUser(String userId); + boolean deleteToken(final String tokenId); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java index ec42a04d7..59f31d5f8 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -5,14 +5,21 @@ import com.flipkart.foxtrot.core.auth.User; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; import javax.inject.Inject; import java.util.Date; @@ -23,7 +30,7 @@ /** * */ - +@Slf4j public class ESAuthStore implements AuthStore { private static final String USERS_INDEX = "user-meta"; private static final String TOKENS_INDEX = "tokens"; @@ -87,20 +94,28 @@ public boolean updateUser( public Optional provisionToken(String userId, TokenType tokenType, Date expiry) { val userPresent = getUser(userId).isPresent(); if (!userPresent) { + log.warn("No user found for is: {}", userId); return Optional.empty(); } final String tokenId = UUID.randomUUID().toString(); - val saveStatus = connection.getClient() - .index(new IndexRequest(TOKENS_INDEX) - .source(mapper.writeValueAsString(new Token(tokenId, tokenType, userId, expiry)), XContentType.JSON) - .id(userId) - .type("TOKEN") - .create(true) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) - .actionGet() - .status(); - if (saveStatus != RestStatus.OK) { - return Optional.empty(); + try { + val saveStatus = connection.getClient() + .index(new IndexRequest(TOKENS_INDEX) + .source(mapper.writeValueAsString(new Token(tokenId, tokenType, userId, expiry)), + XContentType.JSON) + .id(userId) + .type("TOKEN") + .create(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) + .actionGet() + .status(); + if (saveStatus != RestStatus.OK) { + log.error("ES save status for token for user {} is: {}", userId, saveStatus); + return Optional.empty(); + } + } catch (VersionConflictEngineException v) { + log.warn("A valid token exists exists already.. for id: {}", userId); + return getTokenForUser(userId); } return getToken(tokenId); } @@ -118,6 +133,21 @@ public Optional getToken(String tokenId) { return Optional.of(mapper.readValue(getResp.getSourceAsString(), Token.class)); } + @Override + @SneakyThrows + public Optional getTokenForUser(String userId) { + val getResp = connection.getClient() + .search(new SearchRequest(TOKENS_INDEX) + .searchType(SearchType.QUERY_THEN_FETCH) + .source(new SearchSourceBuilder().query(QueryBuilders.termQuery("userId", userId)))) + .actionGet(); + final SearchHits hits = getResp.getHits(); + if (hits.totalHits <= 0) { + return Optional.empty(); + } + return Optional.of(mapper.readValue(hits.getAt(0).getSourceAsString(), Token.class)); + } + @Override public boolean deleteToken(String tokenId) { return connection.getClient() diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java index f56882430..37ff96b67 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java @@ -64,6 +64,7 @@ import javax.inject.Singleton; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; @@ -247,6 +248,9 @@ public JwtConsumer provideJwtConsumer(AuthConfig config) { .setJwsAlgorithmConstraints(new AlgorithmConstraints( AlgorithmConstraints.ConstraintType.WHITELIST, AlgorithmIdentifiers.HMAC_SHA512)) + .setExpectedAudience(Arrays.stream(TokenType.values()) + .map(TokenType::name) + .toArray(String[]::new)) .build(); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java index 9599673eb..217e5e9f3 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java @@ -110,6 +110,13 @@ public Response provisionToken(@NotNull @NotEmpty @PathParam("userId") final Str .build(); } + @GET + @Path("/tokens/{tokenId}") + public Response getToken(@NotNull @NotEmpty @PathParam("tokenId") final String tokenId) { + return Response.ok(authProvider.get().getToken(tokenId)) + .build(); + } + @DELETE @Path("/tokens/{userId}") public Response deleteToken(@NotNull @NotEmpty @PathParam("userId") final String userId, diff --git a/scripts/generate-events.py b/scripts/generate-events.py index 4dcdb93aa..36ba81fe7 100755 --- a/scripts/generate-events.py +++ b/scripts/generate-events.py @@ -45,8 +45,9 @@ def eventBatch(batchSize=100): def postBatch(args): events = eventBatch() - r = requests.post(url="http://" + args.server + "/foxtrot/v1/document/test/bulk", data=json.dumps(events), - headers={'Content-type': 'application/json'}) + headers = headers={'Content-type': 'application/json', 'Authorization' : 'Bearer eyJhbGciOiJIUzUxMiJ9.eyJpc3MiOiJmb3h0cm90LXNlcnZlciIsImp0aSI6Ijc1ODBiODQwLWU2OGEtNDdmOS04OTAwLTkwNmQ1YjAxNmVkYiIsImlhdCI6MTU4NTk5MTE5MSwibmJmIjoxNTg1OTkxMDcxLCJzdWIiOiJwdWJsaXNoZXIiLCJhdWQiOiJTVEFUSUMifQ.xQznh-wH2pSLJNEHeNos_xIfcxgWd9uayOolZvmYj_qdtj7NhO-nwDD_v1ZZ07FS9Z6RKOKWcA6uyjHSUY-R6w'} + print(headers) + r = requests.post(url="http://" + args.server + "/foxtrot/v1/document/test/bulk", data=json.dumps(events), headers = headers) print r if r.status_code == requests.codes.created: print "Sent batch" From 0b4d589fa5b9d0329ded545b4db9442d851812d9 Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Tue, 7 Apr 2020 15:25:34 +0530 Subject: [PATCH 05/37] Hack for consoles history --- .../server/console/ElasticsearchConsolePersistence.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/console/ElasticsearchConsolePersistence.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/console/ElasticsearchConsolePersistence.java index 015da8bb4..c2b7b75cc 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/console/ElasticsearchConsolePersistence.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/console/ElasticsearchConsolePersistence.java @@ -54,7 +54,7 @@ @Singleton public class ElasticsearchConsolePersistence implements ConsolePersistence { - public static final String INDEX_HISTORY = "consoles_history"; + public static final String INDEX_HISTORY = "consoles_v2"; public static final String INDEX_V2 = "consoles_v2"; public static final String INDEX = "consoles"; @@ -154,7 +154,7 @@ public void delete(final String id) { @Override public void saveV2(ConsoleV2 console, boolean newConsole) { - preProcess(console, newConsole); + //preProcess(console, newConsole); try { connection.getClient() .index(new IndexRequest(INDEX_V2) From 6677d73b76bb978cbe74157b2be6bf415d9e6d65 Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Tue, 7 Apr 2020 15:45:40 +0530 Subject: [PATCH 06/37] Local fix --- config/local.yml | 7 ++++--- .../foxtrot/core/querystore/impl/ElasticsearchConfig.java | 4 +++- .../foxtrot/server/resources/ConsoleV2Resource.java | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config/local.yml b/config/local.yml index 2b4e9224a..f01da0b22 100755 --- a/config/local.yml +++ b/config/local.yml @@ -8,15 +8,16 @@ server: elasticsearch: hosts: - - localhost + - stg-els601.phonepe.nm2 cluster: foxtrot tableNamePrefix: foxtrot getQueryTimeout: 10000 + port: 9200 hbase: - secure : false + secure: false tableName: foxtrot - hbaseZookeeperQuorum: localhost:2181 + hbaseZookeeperQuorum: stg-dwhxenial002-stg-cloud005.phonepe.nm2:2181 hbaseZookeeperClientPort: 2181 seggregatedTablePrefix: foxtrot- hbaseZookeeperZnodeParent: /hbase-unsecure diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchConfig.java index def003ac1..a408e0dd4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchConfig.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchConfig.java @@ -16,6 +16,7 @@ package com.flipkart.foxtrot.core.querystore.impl; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Builder; import lombok.NoArgsConstructor; import javax.validation.Valid; @@ -56,7 +57,8 @@ public enum ConnectionType { private long getQueryTimeout; private Integer port; @NotNull - private ConnectionType connectionType; + @Builder.Default + private ConnectionType connectionType = ConnectionType.HTTP; public List getHosts() { return hosts; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java index a320074bb..b109d83f9 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java @@ -77,7 +77,7 @@ public List getList() { @Path("/{id}/old/get") @ApiOperation("get Old Version Console - via id") public ConsoleV2 getOldVersion(@PathParam("id") final String id) { - return consolePersistence.getOldVersion(id); + return consolePersistence.getV2(id); } @GET From 33c62fcb731f8135684ef3f0d4bf583d5126c869 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Tue, 7 Apr 2020 16:20:56 +0530 Subject: [PATCH 07/37] Working jwt header auth --- .../foxtrot/server/auth/AuthStore.java | 2 +- .../foxtrot/server/auth/ESAuthStore.java | 42 +++++----- .../server/auth/TokenAuthenticator.java | 1 + .../foxtrot/server/resources/Auth.java | 2 +- .../foxtrot/server/auth/ESAuthStoreTest.java | 81 +++++++++++++++++++ scripts/generate-events.py | 2 +- 6 files changed, 106 insertions(+), 24 deletions(-) create mode 100644 foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java index 43387fcf0..b339bf2c4 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java @@ -14,7 +14,7 @@ * */ public interface AuthStore { - Optional provision(final User user); + Optional provisionUser(final User user); Optional getUser(final String userId); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java index 41f3f8767..3310c17e1 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -6,6 +6,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; @@ -15,7 +16,6 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHits; @@ -32,8 +32,10 @@ */ @Slf4j public class ESAuthStore implements AuthStore { - private static final String USERS_INDEX = "user-meta"; - private static final String TOKENS_INDEX = "tokens"; + public static final String USERS_INDEX = "user-meta"; + public static final String TOKENS_INDEX = "tokens"; + private static final String TOKEN_TYPE = "TOKEN"; + private static final String USER_TYPE = "USER"; private final ElasticsearchConnection connection; private final ObjectMapper mapper; @@ -46,9 +48,9 @@ public ESAuthStore(ElasticsearchConnection connection, ObjectMapper mapper) { @Override @SneakyThrows - public Optional provision(User user) { + public Optional provisionUser(User user) { val status = saveUser(user, DocWriteRequest.OpType.CREATE); - if (status != RestStatus.OK) { + if (status != RestStatus.CREATED) { return Optional.empty(); } return getUser(user.getId()); @@ -58,8 +60,7 @@ public Optional provision(User user) { @SneakyThrows public Optional getUser(String userId) { val getResp = connection.getClient() - .get(new GetRequest(USERS_INDEX) - .id(userId), RequestOptions.DEFAULT); + .get(new GetRequest(USERS_INDEX, USER_TYPE, userId), RequestOptions.DEFAULT); if (!getResp.isExists()) { return Optional.empty(); } @@ -70,8 +71,7 @@ public Optional getUser(String userId) { @Override public boolean deleteUser(String id) { return connection.getClient() - .delete(new DeleteRequest(USERS_INDEX) - .id(id) + .delete(new DeleteRequest(USERS_INDEX, USER_TYPE, id) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) .status() == RestStatus.OK; } @@ -85,7 +85,7 @@ public boolean updateUser( return false; } final User updatedUser = mutator.apply(user); - return saveUser(updatedUser, DocWriteRequest.OpType.UPDATE) == RestStatus.OK; + return saveUser(updatedUser, DocWriteRequest.OpType.INDEX) == RestStatus.OK; } @Override @@ -102,16 +102,16 @@ public Optional provisionToken(String userId, TokenType tokenType, Date e .index(new IndexRequest(TOKENS_INDEX) .source(mapper.writeValueAsString(new Token(tokenId, tokenType, userId, expiry)), XContentType.JSON) - .id(userId) - .type("TOKEN") + .id(tokenId) + .type(TOKEN_TYPE) .create(true) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) .status(); - if (saveStatus != RestStatus.OK) { + if (saveStatus != RestStatus.CREATED) { log.error("ES save status for token for user {} is: {}", userId, saveStatus); return Optional.empty(); } - } catch (VersionConflictEngineException v) { + } catch (ElasticsearchException v) { log.warn("A valid token exists exists already.. for id: {}", userId); return getTokenForUser(userId); } @@ -122,8 +122,7 @@ public Optional provisionToken(String userId, TokenType tokenType, Date e @SneakyThrows public Optional getToken(String tokenId) { val getResp = connection.getClient() - .get(new GetRequest(TOKENS_INDEX) - .id(tokenId), RequestOptions.DEFAULT); + .get(new GetRequest(TOKENS_INDEX, TOKEN_TYPE, tokenId), RequestOptions.DEFAULT); if (!getResp.isExists()) { return Optional.empty(); } @@ -135,8 +134,10 @@ public Optional getToken(String tokenId) { public Optional getTokenForUser(String userId) { val getResp = connection.getClient() .search(new SearchRequest(TOKENS_INDEX) - .searchType(SearchType.QUERY_THEN_FETCH) - .source(new SearchSourceBuilder().query(QueryBuilders.termQuery("userId", userId))), + .searchType(SearchType.QUERY_THEN_FETCH) + .types(TOKEN_TYPE) + .source(new SearchSourceBuilder().query( + QueryBuilders.termQuery("userId", userId))), RequestOptions.DEFAULT); final SearchHits hits = getResp.getHits(); if (hits.totalHits <= 0) { @@ -149,8 +150,7 @@ public Optional getTokenForUser(String userId) { @SneakyThrows public boolean deleteToken(String tokenId) { return connection.getClient() - .delete(new DeleteRequest(TOKENS_INDEX) - .id(tokenId) + .delete(new DeleteRequest(TOKENS_INDEX, TOKEN_TYPE, tokenId) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) .status() == RestStatus.OK; } @@ -162,7 +162,7 @@ private RestStatus saveUser(User user, DocWriteRequest.OpType opType) { .index(new IndexRequest(USERS_INDEX) .source(mapper.writeValueAsString(user), XContentType.JSON) .id(user.getId()) - .type("USER") + .type(USER_TYPE) .opType(opType) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java index 22521ad68..a01dbd3cb 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java @@ -33,6 +33,7 @@ public Optional authenticate(JwtContext jwtContext) throws Authen log.debug("Authentication is disabled"); return Optional.of(UserPrincipal.DEFAULT); } + log.debug("Auth called"); final String userId; final String tokenId; try { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java index 217e5e9f3..fb1e82502 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java @@ -43,7 +43,7 @@ public Auth(Provider authProvider, AuthConfig authConfig) { public Response provisionUser(@NotNull @Valid final CreateUserRequest createUserRequest) { val user = new User(createUserRequest.getId(), createUserRequest.getRoles(), createUserRequest.getTables(), new Date(), new Date()); - return Response.ok(authProvider.get().provision(user)).build(); + return Response.ok(authProvider.get().provisionUser(user)).build(); } @GET diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java new file mode 100644 index 000000000..66217bf1a --- /dev/null +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java @@ -0,0 +1,81 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.TestUtils; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import com.flipkart.foxtrot.core.table.impl.ElasticsearchTestUtils; +import com.google.common.collect.Sets; +import io.dropwizard.jackson.Jackson; +import lombok.SneakyThrows; +import lombok.val; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Objects; + +/** + * + */ +public class ESAuthStoreTest { + + private ElasticsearchConnection elasticsearchConnection; + private AuthStore authStore; + + @Before + public void setup() throws Exception { + elasticsearchConnection = ElasticsearchTestUtils.getConnection(); + TestUtils.ensureIndex(elasticsearchConnection, ESAuthStore.USERS_INDEX); + TestUtils.ensureIndex(elasticsearchConnection, ESAuthStore.TOKENS_INDEX); + authStore = new ESAuthStore(elasticsearchConnection, Jackson.newObjectMapper()); + } + + @Test + @SneakyThrows + public void usersTest() { + final String TEST_USER = "test_user"; + final User originalUser = new User(TEST_USER, + EnumSet.of(FoxtrotRole.INGEST), + Collections.emptySet(), + null, + null); + Assert.assertEquals(originalUser, authStore.provisionUser(originalUser).orElse(null)); + Assert.assertEquals(originalUser, authStore.getUser(TEST_USER).orElse(null)); + Assert.assertTrue(authStore.grantRole(TEST_USER, FoxtrotRole.QUERY)); + Assert.assertTrue(Sets.difference(EnumSet.of(FoxtrotRole.INGEST, FoxtrotRole.QUERY), + Objects.requireNonNull(authStore.getUser(TEST_USER) + .map(User::getRoles) + .orElse(null))).isEmpty()); + Assert.assertTrue(authStore.revokeRole(TEST_USER, FoxtrotRole.INGEST)); + Assert.assertTrue(Sets.difference(EnumSet.of(FoxtrotRole.QUERY), + Objects.requireNonNull(authStore.getUser(TEST_USER) + .map(User::getRoles) + .orElse(null))).isEmpty()); + Assert.assertTrue(authStore.deleteUser(TEST_USER)); + } + + @Test + public void tokensTest() { + final String TEST_USER = "test_user"; + Assert.assertNull(authStore.provisionToken(TEST_USER, TokenType.STATIC, null) + .orElse(null)); + final User originalUser = new User(TEST_USER, + EnumSet.of(FoxtrotRole.INGEST), + Collections.emptySet(), null, null); + Assert.assertEquals(originalUser, authStore.provisionUser(originalUser).orElse(null)); + val token = authStore.provisionToken(TEST_USER, TokenType.STATIC, null) + .orElse(null); + Assert.assertNotNull(token); + Assert.assertEquals(TEST_USER, token.getUserId()); + Assert.assertEquals(token.getId(), + Objects.requireNonNull(authStore.getToken(token.getId()).orElse(null)).getId()); + Assert.assertEquals(token.getId(), + Objects.requireNonNull(authStore.getTokenForUser(TEST_USER).orElse(null)).getId()); + Assert.assertTrue(authStore.deleteToken(token.getId())); + Assert.assertNull(authStore.getToken(token.getId()).orElse(null)); + Assert.assertTrue(authStore.deleteUser(TEST_USER)); + } +} \ No newline at end of file diff --git a/scripts/generate-events.py b/scripts/generate-events.py index 36ba81fe7..1641edd95 100755 --- a/scripts/generate-events.py +++ b/scripts/generate-events.py @@ -45,7 +45,7 @@ def eventBatch(batchSize=100): def postBatch(args): events = eventBatch() - headers = headers={'Content-type': 'application/json', 'Authorization' : 'Bearer eyJhbGciOiJIUzUxMiJ9.eyJpc3MiOiJmb3h0cm90LXNlcnZlciIsImp0aSI6Ijc1ODBiODQwLWU2OGEtNDdmOS04OTAwLTkwNmQ1YjAxNmVkYiIsImlhdCI6MTU4NTk5MTE5MSwibmJmIjoxNTg1OTkxMDcxLCJzdWIiOiJwdWJsaXNoZXIiLCJhdWQiOiJTVEFUSUMifQ.xQznh-wH2pSLJNEHeNos_xIfcxgWd9uayOolZvmYj_qdtj7NhO-nwDD_v1ZZ07FS9Z6RKOKWcA6uyjHSUY-R6w'} + headers = headers={'Content-type': 'application/json', 'Authorization' : 'Bearer eyJhbGciOiJIUzUxMiJ9.eyJpc3MiOiJmb3h0cm90LXNlcnZlciIsImp0aSI6IjA5YzIwMmRhLTg2YjctNGMyZS1iMzA1LWZjNTBkNjliYTRlNSIsImlhdCI6MTU4NjI1NDE2NSwibmJmIjoxNTg2MjU0MDQ1LCJzdWIiOiJldmVudF9wdWJsaXNoZXIiLCJhdWQiOiJTVEFUSUMifQ.jGijnjI8TSXJ7F2OnwE-r5HuN_YRfvoG45umga-vVhI_Qeu83k17AtBbeGbaaTT8sPJDQFIktz4EGe7TWkLUvw'} print(headers) r = requests.post(url="http://" + args.server + "/foxtrot/v1/document/test/bulk", data=json.dumps(events), headers = headers) print r From eff49111158be797280f665de32e15dae23445d4 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 9 Apr 2020 03:03:49 +0530 Subject: [PATCH 08/37] Google auth --- .gitignore | 1 + config/docker.yml | 15 +- docker-compose-auth.yml | 63 +++++++ docker-compose.yml | 5 + .../foxtrot/core/auth/FoxtrotRole.java | 5 +- .../com/flipkart/foxtrot/core/auth/User.java | 3 +- foxtrot-server/pom.xml | 20 ++ .../server/GlobalAuthCheckingFilter.java | 87 +++++++++ .../foxtrot/server/auth/AuthConfig.java | 8 +- .../foxtrot/server/auth/AuthStore.java | 18 +- .../foxtrot/server/auth/ESAuthStore.java | 54 +++++- .../server/auth/JWTAuthenticationFailure.java | 9 + .../server/auth/JwtAuthDynamicFeature.java | 8 +- .../foxtrot/server/auth/JwtAuthFilter.java | 171 ++++++++++++++++++ .../foxtrot/server/auth/JwtConfig.java | 3 + .../server/auth/TokenAuthenticator.java | 2 +- .../foxtrot/server/auth/TokenType.java | 2 +- .../auth/authprovider/AuthConfigVisitor.java | 10 + .../auth/authprovider/AuthProvider.java | 31 ++++ .../auth/authprovider/AuthProviderConfig.java | 31 ++++ .../authprovider/AuthProviderFactory.java | 10 + .../server/auth/authprovider/AuthType.java | 8 + .../authprovider/impl/GoogleAuthProvider.java | 159 ++++++++++++++++ .../impl/GoogleAuthProviderConfig.java | 75 ++++++++ .../server/auth/io/CreateUserRequest.java | 1 + .../DistributedSessionDataStore.java | 84 +++++++++ .../auth/sessionstore/SessionDataStore.java | 12 ++ .../config/FoxtrotServerConfiguration.java | 6 + .../foxtrot/server/di/FoxtrotModule.java | 27 ++- .../ExpiredSessionsCleaner.java | 55 ++++++ .../sessioncleanup/SessionCleanupConfig.java | 15 ++ .../AuthenticationExceptionHandler.java | 25 +++ .../server/resources/AnalyticsResource.java | 2 +- .../server/resources/AsyncResource.java | 3 + .../foxtrot/server/resources/Auth.java | 4 +- .../server/resources/CacheUpdateResource.java | 2 + .../resources/ClusterHealthResource.java | 2 + .../server/resources/ClusterInfoResource.java | 2 + .../server/resources/ConsoleResource.java | 1 + .../server/resources/ConsoleV2Resource.java | 7 + .../foxtrot/server/resources/GoogleAuth.java | 118 ++++++++++++ .../resources/TableFieldMappingResource.java | 5 + .../resources/TableManagerResource.java | 45 +++-- .../server/resources/UtilResource.java | 22 ++- .../foxtrot/server/utils/AuthUtils.java | 9 + .../foxtrot/server/ResourceTestUtils.java | 30 +++ .../foxtrot/server/auth/DummyAuthFilter.java | 45 +++++ .../foxtrot/server/auth/ESAuthStoreTest.java | 53 +++++- .../resources/AnalyticsResourceTest.java | 21 +-- .../server/resources/AsyncResourceTest.java | 18 +- .../resources/CacheUpdateResourceTest.java | 9 +- .../resources/ClusterHealthResourceTest.java | 12 +- .../resources/DocumentResourceTest.java | 48 +++-- .../server/resources/FoxtrotResourceTest.java | 2 +- .../TableFieldMappingResourceTest.java | 18 +- .../resources/TableManagerResourceTest.java | 20 +- 56 files changed, 1383 insertions(+), 138 deletions(-) create mode 100644 docker-compose-auth.yml create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/GlobalAuthCheckingFilter.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JWTAuthenticationFailure.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProviderConfig.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/SessionDataStore.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/ExpiredSessionsCleaner.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/SessionCleanupConfig.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/GoogleAuth.java create mode 100644 foxtrot-server/src/test/java/com/flipkart/foxtrot/server/ResourceTestUtils.java create mode 100644 foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/DummyAuthFilter.java diff --git a/.gitignore b/.gitignore index 7a6589d59..814b05682 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ .DS_Store target/* config/test.yml +gauth-creds-env.sh diff --git a/config/docker.yml b/config/docker.yml index a196c2eb4..58abe67aa 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -67,8 +67,21 @@ consoleHistoryConfig: interval: 86400 initialDelay: 10 +sessionCleanupConfig: + active: true + interval: 86400 + initialDelay: 10 + auth: - disabled: false + enabled: ${AUTH_ENABLED} jwt: issuerId: foxtrot-server privateKey: "6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" + sessionDuration: 1h + + provider: + type: OAUTH_GOOGLE + clientId: ${GOOGLE_CLIENT_ID} + clientSecret: ${GOOGLE_CLIENT_SECRET} + server: ${GOOGLE_CALLBACK_HOST_PORT} + secureEndpoint: ${GOOGLE_SECURE_ENDPOINT} diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml new file mode 100644 index 000000000..9bc83fd52 --- /dev/null +++ b/docker-compose-auth.yml @@ -0,0 +1,63 @@ +version: '2' +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.8 + container_name: elasticsearch + hostname: elasticsearch + environment: + - cluster.name=elasticsearch + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + command: elasticsearch -Ehttp.cors.enabled=true + volumes: + - /elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + + hbase: + image: hyness/hbase-rest-standalone + hostname: hbase + ports: + - "2181:2181" + - "8080:8080" + - "16000:16000" + - "16010:16010" + - "16020:16020" + - "16030:16030" + volumes: + - /data/hbase + - /data/zookeeper + + foxtrot-server: + depends_on: + - elasticsearch + - hbase + container_name: foxtrot_server + build: . + ports: + - "17000:17000" + - "17001:17001" + - "5701:5701" + volumes: + - /var/log/foxtrot-server + environment: + - HOST=localhost + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_CLUSTER_NAME=elasticsearch + - ELASTICSEARCH_TABLE_NAME_PREFIX=foxtrot + - ELASTICSEARCH_PORT="9200" + - ELASTICSEARCH_PROTOCOL=http + - HBASE_TABLE_NAME=foxtrot + - HBASE_ZOOKEEPER_QUORUM=hbase + - HBASE_ZOOKEEPER_CLIENT_PORT=2181 + - HAZELCAST_CLUSTER_NAME=foxtrot + - AUTH_ENABLED=true + - GOOGLE_CALLBACK_HOST_PORT=localhost:17000 + - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} + - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} + - GOOGLE_SECURE_ENDPOINT=false diff --git a/docker-compose.yml b/docker-compose.yml index 9ace91907..e48275b29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,3 +56,8 @@ services: - HBASE_ZOOKEEPER_QUORUM=hbase - HBASE_ZOOKEEPER_CLIENT_PORT=2181 - HAZELCAST_CLUSTER_NAME=foxtrot + - AUTH_ENABLED=false + - GOOGLE_CALLBACK_HOST_PORT=localhost:17000 + - GOOGLE_CLIENT_ID=dummy + - GOOGLE_CLIENT_SECRET=dummy + - GOOGLE_SECURE_ENDPOINT=false diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java index 188f140a9..ed2723a95 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java @@ -9,7 +9,9 @@ public enum FoxtrotRole { INGEST(Value.INGEST), CONSOLE(Value.CONSOLE), - QUERY(Value.QUERY); + QUERY(Value.QUERY), + SYSADMIN(Value.SYSADMIN), + ; FoxtrotRole(String value) { this.value = value; @@ -19,6 +21,7 @@ public static class Value { public static final String INGEST = "INGEST"; public static final String CONSOLE = "CONSOLE"; public static final String QUERY = "QUERY"; + public static final String SYSADMIN = "SYSADMIN"; } private final String value; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java index b4291f5ab..5a946a953 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java @@ -13,11 +13,12 @@ @Value public class User { public static final User DEFAULT - = new User("__DEFAULT__", EnumSet.allOf(FoxtrotRole.class), Collections.emptySet(), new Date(), new Date()); + = new User("__DEFAULT__", EnumSet.allOf(FoxtrotRole.class), Collections.emptySet(), true, new Date(), new Date()); String id; Set roles; Set tables; + boolean systemUser; Date created; Date updated; } diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 760d238ed..a6d5dfad0 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -100,6 +100,11 @@ opencsv 2.0 + + com.google.api-client + google-api-client + 1.23.0 + com.hazelcast hazelcast @@ -107,6 +112,21 @@ test-jar test + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + 2.25.1 + + + javax.servlet + javax.servlet-api + + + junit + junit + + + diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/GlobalAuthCheckingFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/GlobalAuthCheckingFilter.java new file mode 100644 index 000000000..b9994efa8 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/GlobalAuthCheckingFilter.java @@ -0,0 +1,87 @@ +package com.flipkart.foxtrot.server; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import javax.annotation.Priority; +import javax.inject.Inject; +import javax.inject.Provider; +import javax.servlet.*; +import javax.servlet.annotation.WebFilter; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Priorities; +import javax.ws.rs.core.HttpHeaders; +import java.io.IOException; +import java.util.Arrays; +import java.util.Set; + +/** + * + */ +@Priority(Priorities.AUTHENTICATION) +@WebFilter("/*") +@Slf4j +public class GlobalAuthCheckingFilter implements Filter { + private static final Set WHITELISTED_PATTERNS = ImmutableSet.builder() + .add("/foxtrot/google") + .add("^/foxtrot/auth.*") + .build(); + private final AuthConfig authConfig; + private final Provider sessionDataStore; + + @Inject + public GlobalAuthCheckingFilter( + AuthConfig authConfig, + Provider sessionDataStore) { + this.authConfig = authConfig; + this.sessionDataStore = sessionDataStore; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + + } + + @Override + public void doFilter( + ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + if(!authConfig.isEnabled()) { + log.trace("Auth disabled"); + chain.doFilter(request, response); + return; + } + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + final String requestURI = httpRequest.getRequestURI(); + if(WHITELISTED_PATTERNS.stream().anyMatch(requestURI::startsWith)) { + chain.doFilter(request, response); + return; + } + val authHeader = httpRequest.getHeader(HttpHeaders.AUTHORIZATION); + val cookies = httpRequest.getCookies(); + if(!Strings.isNullOrEmpty(authHeader)) { + chain.doFilter(request, response); + return; + } + if(null != cookies && cookies.length != 0 + && Arrays.stream(cookies).anyMatch(cookie -> cookie.getName().equals("token"))) { + chain.doFilter(request, response); + return; + } + val referrer = httpRequest.getHeader(org.apache.http.HttpHeaders.REFERER); + val source = Strings.isNullOrEmpty(referrer) ? requestURI : referrer; + httpResponse.addCookie(new Cookie("redirection", source)); + httpResponse.sendRedirect("/foxtrot/google/login"); + } + + @Override + public void destroy() { + + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java index ac1b22761..84c28f4fc 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java @@ -1,5 +1,6 @@ package com.flipkart.foxtrot.server.auth; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; import lombok.Data; import javax.validation.Valid; @@ -10,8 +11,13 @@ */ @Data public class AuthConfig { - private boolean disabled; + private boolean enabled; + @NotNull @Valid private JwtConfig jwt = new JwtConfig(); + + @NotNull + @Valid + public AuthProviderConfig provider; } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java index b339bf2c4..c55b820f5 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java @@ -2,12 +2,14 @@ import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.auth.User; +import io.dropwizard.util.Duration; import lombok.SneakyThrows; import lombok.val; import java.util.Date; import java.util.HashSet; import java.util.Optional; +import java.util.UUID; import java.util.function.UnaryOperator; /** @@ -26,7 +28,7 @@ default boolean grantRole(final String userId, final FoxtrotRole role) { return updateUser(userId, user -> { val roles = user.getRoles() == null ? new HashSet() : user.getRoles(); roles.add(role); - return new User(userId, roles, user.getTables(), user.getCreated(), new Date()); + return new User(userId, roles, user.getTables(), user.isSystemUser(), user.getCreated(), new Date()); }); } @@ -34,7 +36,7 @@ default boolean revokeRole(final String userId, final FoxtrotRole role) { return updateUser(userId, user -> { val roles = user.getRoles() == null ? new HashSet() : user.getRoles(); roles.remove(role); - return new User(userId, roles, user.getTables(), user.getCreated(), new Date()); + return new User(userId, roles, user.getTables(), user.isSystemUser(), user.getCreated(), new Date()); }); } @@ -42,7 +44,7 @@ default boolean grantTableAccess(final String userId, final String table) { return updateUser(userId, user -> { val tables = user.getTables() == null ? new HashSet() : user.getTables(); tables.add(table); - return new User(userId, user.getRoles(), tables, user.getCreated(), new Date()); + return new User(userId, user.getRoles(), tables, user.isSystemUser(), user.getCreated(), new Date()); }); } @@ -50,11 +52,15 @@ default boolean revokeTableAccess(final String userId, final String table) { return updateUser(userId, user -> { val tables = user.getTables() == null ? new HashSet() : user.getTables(); tables.remove(table); - return new User(userId, user.getRoles(), tables, user.getCreated(), new Date()); + return new User(userId, user.getRoles(), tables, user.isSystemUser(), user.getCreated(), new Date()); }); } - Optional provisionToken(final String userId, TokenType tokenType, Date expiry); + default Optional provisionToken(final String userId, TokenType tokenType, Date expiry) { + return provisionToken(userId, UUID.randomUUID().toString(), tokenType, expiry); + } + + Optional provisionToken(final String userId, String tokenId, TokenType tokenType, Date expiry); Optional getToken(final String tokenId); @@ -62,4 +68,6 @@ default boolean revokeTableAccess(final String userId, final String table) { Optional getTokenForUser(String userId); boolean deleteToken(final String tokenId); + + boolean deleteExpiredTokens(Date date, Duration sessionDuration); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java index 3310c17e1..8100bd549 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -2,7 +2,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.core.querystore.actions.Utils; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import io.dropwizard.util.Duration; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -17,6 +19,7 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -24,7 +27,6 @@ import javax.inject.Inject; import java.util.Date; import java.util.Optional; -import java.util.UUID; import java.util.function.UnaryOperator; /** @@ -70,10 +72,15 @@ public Optional getUser(String userId) { @SneakyThrows @Override public boolean deleteUser(String id) { - return connection.getClient() + boolean status = connection.getClient() .delete(new DeleteRequest(USERS_INDEX, USER_TYPE, id) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) .status() == RestStatus.OK; + if (status) { + val count = deleteTokensForUser(id); + log.debug("User {} deleted and {} existing tokens deleted with that.", id, count); + } + return status; } @Override @@ -90,13 +97,18 @@ public boolean updateUser( @Override @SneakyThrows - public Optional provisionToken(String userId, TokenType tokenType, Date expiry) { - val userPresent = getUser(userId).isPresent(); - if (!userPresent) { + public Optional provisionToken(String userId, String tokenId, TokenType tokenType, Date expiry) { + val user = getUser(userId).orElse(null); + if (null == user) { log.warn("No user found for is: {}", userId); return Optional.empty(); } - final String tokenId = UUID.randomUUID().toString(); + if ((tokenType.equals(TokenType.STATIC) + || tokenType.equals(TokenType.SYSTEM)) + && !user.isSystemUser()) { + log.warn("Cannot create system/static token for non system user"); + return Optional.empty(); + } try { val saveStatus = connection.getClient() .index(new IndexRequest(TOKENS_INDEX) @@ -111,7 +123,9 @@ public Optional provisionToken(String userId, TokenType tokenType, Date e log.error("ES save status for token for user {} is: {}", userId, saveStatus); return Optional.empty(); } - } catch (ElasticsearchException v) { + log.info("Token created for user: {}", userId); + } + catch (ElasticsearchException v) { log.warn("A valid token exists exists already.. for id: {}", userId); return getTokenForUser(userId); } @@ -155,6 +169,22 @@ public boolean deleteToken(String tokenId) { .status() == RestStatus.OK; } + @Override + @SneakyThrows + public boolean deleteExpiredTokens(Date date, Duration sessionDuration) { + log.info("Cleaning up sessions older than: {}", sessionDuration); + Date oldestValidDate = new Date(date.getTime() - sessionDuration.toMilliseconds()); + val deletedCount = connection.getClient() + .deleteByQuery(new DeleteByQueryRequest(TOKENS_INDEX) + .setDocTypes(TOKEN_TYPE) + .setIndicesOptions(Utils.indicesOptions()) + .setQuery(QueryBuilders.rangeQuery("expiry") + .lt(oldestValidDate.getTime())), RequestOptions.DEFAULT) + .getDeleted(); + log.info("Deleted {} expired tokens", deletedCount); + return true; + } + @SneakyThrows private RestStatus saveUser(User user, DocWriteRequest.OpType opType) { @@ -168,4 +198,14 @@ private RestStatus saveUser(User user, DocWriteRequest.OpType opType) { RequestOptions.DEFAULT) .status(); } + + @SneakyThrows + public long deleteTokensForUser(String userId) { + return connection.getClient() + .deleteByQuery(new DeleteByQueryRequest(TOKENS_INDEX) + .setDocTypes(TOKEN_TYPE) + .setQuery(QueryBuilders.termQuery("userId", userId)), + RequestOptions.DEFAULT) + .getDeleted(); + } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JWTAuthenticationFailure.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JWTAuthenticationFailure.java new file mode 100644 index 000000000..59a3d7ee5 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JWTAuthenticationFailure.java @@ -0,0 +1,9 @@ +package com.flipkart.foxtrot.server.auth; + +import java.io.IOException; + +/** + * + */ +public class JWTAuthenticationFailure extends IOException { +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java index f652e7c82..5baf83d9a 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java @@ -1,6 +1,5 @@ package com.flipkart.foxtrot.server.auth; -import com.github.toastshaman.dropwizard.auth.jwt.JwtAuthFilter; import io.dropwizard.auth.*; import io.dropwizard.setup.Environment; import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature; @@ -17,18 +16,21 @@ public class JwtAuthDynamicFeature extends AuthDynamicFeature { @Inject public JwtAuthDynamicFeature( + AuthConfig authConfig, JwtConsumer jwtConsumer, Authorizer authorizer, Authenticator authenticator, Environment environment) { - super(new JwtAuthFilter.Builder() + super(new JwtAuthFilter.Builder() + .setAuthConfig(authConfig) .setJwtConsumer(jwtConsumer) + .setCookieName("token") .setPrefix("Bearer") .setRealm("realm") .setAuthenticator(authenticator) .setAuthorizer(authorizer) .setUnauthorizedHandler(new DefaultUnauthorizedHandler()) - .buildAuthFilter()); + .buildAuthFilter()); if(null != environment) { environment.jersey().register(new AuthValueFactoryProvider.Binder<>(UserPrincipal.class)); environment.jersey().register(RolesAllowedDynamicFeature.class); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java new file mode 100644 index 000000000..af26759b6 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java @@ -0,0 +1,171 @@ +package com.flipkart.foxtrot.server.auth; + +import io.dropwizard.auth.AuthFilter; +import io.dropwizard.auth.AuthenticationException; +import io.dropwizard.auth.Authenticator; +import lombok.val; +import org.jose4j.jwt.consumer.InvalidJwtException; +import org.jose4j.jwt.consumer.JwtConsumer; +import org.jose4j.jwt.consumer.JwtContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Priority; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.Priorities; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.core.Cookie; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.SecurityContext; +import java.io.IOException; +import java.security.Principal; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkNotNull; +import static javax.ws.rs.core.HttpHeaders.AUTHORIZATION; + +@Priority(Priorities.AUTHENTICATION) +public class JwtAuthFilter extends AuthFilter { + + private static final Logger LOGGER = LoggerFactory.getLogger(JwtAuthFilter.class); + + private final AuthConfig authConfig; + private final JwtConsumer consumer; + private final String cookieName; + + private JwtAuthFilter( + AuthConfig authConfig, + JwtConsumer consumer, + String cookieName) { + this.authConfig = authConfig; + this.consumer = consumer; + this.cookieName = cookieName; + } + + @Override + public void filter(final ContainerRequestContext requestContext) throws IOException { + LOGGER.info("AUTH Called"); + if(!authConfig.isEnabled()) { + updateContext(requestContext, UserPrincipal.DEFAULT); + return; + } + final Optional optionalToken = getTokenFromCookieOrHeader(requestContext); + + if (optionalToken.isPresent()) { + try { + val jwtContext = verifyToken(optionalToken.get()); + val principal = authenticator.authenticate(jwtContext); + + if (principal.isPresent()) { + updateContext(requestContext, principal.get()); + return; + } + } catch (InvalidJwtException ex) { + LOGGER.warn("Error decoding credentials: " + ex.getMessage(), ex); + } catch (AuthenticationException ex) { + LOGGER.warn("Error authenticating credentials", ex); + throw new InternalServerErrorException(); + } + } + + throw new JWTAuthenticationFailure(); +// throw new WebApplicationException(unauthorizedHandler.buildResponse(prefix, realm)); + } + + private void updateContext(ContainerRequestContext requestContext, UserPrincipal principal) { + requestContext.setSecurityContext(new SecurityContext() { + + @Override + public Principal getUserPrincipal() { + return principal; + } + + @Override + public boolean isUserInRole(String role) { + return authorizer.authorize(principal, role); + } + + @Override + public boolean isSecure() { + return requestContext.getSecurityContext().isSecure(); + } + + @Override + public String getAuthenticationScheme() { + return SecurityContext.BASIC_AUTH; + } + + }); + } + + private JwtContext verifyToken(String rawToken) throws InvalidJwtException { + return consumer.process(rawToken); + } + + private Optional getTokenFromCookieOrHeader(ContainerRequestContext requestContext) { + final Optional headerToken = getTokenFromHeader(requestContext.getHeaders()); + return headerToken.isPresent() ? headerToken : getTokenFromCookie(requestContext); + } + + private Optional getTokenFromHeader(MultivaluedMap headers) { + final String header = headers.getFirst(AUTHORIZATION); + if (header != null) { + int space = header.indexOf(' '); + if (space > 0) { + final String method = header.substring(0, space); + if (prefix.equalsIgnoreCase(method)) { + final String rawToken = header.substring(space + 1); + return Optional.of(rawToken); + } + } + } + + return Optional.empty(); + } + + private Optional getTokenFromCookie(ContainerRequestContext requestContext) { + final Map cookies = requestContext.getCookies(); + + if (cookieName != null && cookies.containsKey(cookieName)) { + final Cookie tokenCookie = cookies.get(cookieName); + final String rawToken = tokenCookie.getValue(); + return Optional.of(rawToken); + } + + return Optional.empty(); + } + + /** + * Builder for {@link JwtAuthFilter}. + *

An {@link Authenticator} must be provided during the building process.

+ * + */ + public static class Builder extends AuthFilterBuilder { + + private AuthConfig authConfig; + private JwtConsumer consumer; + private String cookieName; + + public Builder setAuthConfig(AuthConfig authConfig) { + this.authConfig = authConfig; + return this; + } + + public Builder setJwtConsumer(JwtConsumer consumer) { + this.consumer = consumer; + return this; + } + + public Builder setCookieName(String cookieName) { + this.cookieName = cookieName; + return this; + } + + @Override + protected JwtAuthFilter newInstance() { + checkNotNull(consumer, "JwtConsumer is not set"); + return new JwtAuthFilter(authConfig, consumer, cookieName); + } + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java index 3c13ece5e..db2835598 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java @@ -1,6 +1,7 @@ package com.flipkart.foxtrot.server.auth; import com.google.common.annotations.VisibleForTesting; +import io.dropwizard.util.Duration; import lombok.Data; import lombok.NoArgsConstructor; import org.hibernate.validator.constraints.NotEmpty; @@ -25,6 +26,8 @@ public class JwtConfig { @NotNull private String authCachePolicy = "maximumSize=10000, expireAfterAccess=10m"; + private Duration sessionDuration = Duration.days(30); + @VisibleForTesting public JwtConfig(String privateKey, String issuerId) { this.privateKey = privateKey; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java index a01dbd3cb..b2931cf1f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java @@ -29,7 +29,7 @@ public TokenAuthenticator(AuthConfig config, final Provider authStore @Override public Optional authenticate(JwtContext jwtContext) throws AuthenticationException { - if(config.isDisabled()) { + if(!config.isEnabled()) { log.debug("Authentication is disabled"); return Optional.of(UserPrincipal.DEFAULT); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java index 27a334c15..7369e86ef 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java @@ -11,7 +11,7 @@ */ @Getter public enum TokenType { - DYNAMIC(EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.CONSOLE)), + DYNAMIC(EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.CONSOLE, FoxtrotRole.SYSADMIN)), STATIC(EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.INGEST)), SYSTEM(EnumSet.allOf(FoxtrotRole.class)); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java new file mode 100644 index 000000000..ddf5b4886 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java @@ -0,0 +1,10 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; + +/** + * + */ +public interface AuthConfigVisitor { + T visit(GoogleAuthProviderConfig googleAuthConfig); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java new file mode 100644 index 000000000..8d526bbbc --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java @@ -0,0 +1,31 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.flipkart.foxtrot.server.auth.Token; +import lombok.Builder; +import lombok.Data; + +import java.util.Optional; + +/** + * + */ +public interface AuthProvider { + + + @Data + @Builder + public class AuthInfo { + private final String ip; + private final String email; + } + + AuthType type(); + + String redirectionURL(String sessionId); + + default boolean isPreregistrationRequired() { + return true; + } + + Optional login(String authCode, String sessionId); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java new file mode 100644 index 000000000..97d918eb5 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java @@ -0,0 +1,31 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; +import lombok.Data; + +/** + * + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(name = "OAUTH_GOOGLE", value = GoogleAuthProviderConfig.class), +}) +@Data +public abstract class AuthProviderConfig { + private final AuthType type; + + private boolean enabled; + + protected AuthProviderConfig(AuthType type) { + this.type = type; + } + + protected AuthProviderConfig(AuthType type, boolean enabled) { + this(type); + this.enabled = enabled; + } + + abstract public T accept(AuthConfigVisitor visitor); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java new file mode 100644 index 000000000..9435b56e4 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java @@ -0,0 +1,10 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import java.util.Map; + +/** + * + */ +public interface AuthProviderFactory { + Map build(); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java new file mode 100644 index 000000000..71e18aebe --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java @@ -0,0 +1,8 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +/** + * + */ +public enum AuthType { + OAUTH_GOOGLE +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java new file mode 100644 index 000000000..d2ae5a5f2 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java @@ -0,0 +1,159 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.flipkart.foxtrot.server.auth.Token; +import com.flipkart.foxtrot.server.auth.TokenType; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; +import com.flipkart.foxtrot.server.utils.AuthUtils; +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeTokenRequest; +import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Inject; +import io.dropwizard.util.Duration; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.util.Date; +import java.util.Optional; + +/** + * + */ +@Slf4j +public class GoogleAuthProvider implements AuthProvider { + + public static final String CALLBACK_PATH = "/foxtrot/google/callback"; + + private final HttpTransport transport; + private final GoogleAuthorizationCodeFlow authorizationCodeFlow; + private final String redirectionUrl; + private final AuthConfig authConfig; + private final GoogleAuthProviderConfig googleAuthConfig; + private final ObjectMapper mapper; + private final AuthStore credentialsStorage; + + @Inject + public GoogleAuthProvider( + GoogleAuthProviderConfig googleAuthConfig, + AuthConfig authConfig, ObjectMapper mapper, + AuthStore credentialsStorage) { + this.authConfig = authConfig; + final NetHttpTransport.Builder transportBuilder = new NetHttpTransport.Builder(); + Proxy proxy = Proxy.NO_PROXY; + if (googleAuthConfig.getProxyType() != null) { + switch (googleAuthConfig.getProxyType()) { + case DIRECT: + break; + case HTTP: { + Preconditions.checkArgument(!Strings.isNullOrEmpty(googleAuthConfig.getProxyHost())); + proxy = new Proxy(Proxy.Type.HTTP, + new InetSocketAddress(googleAuthConfig.getProxyHost(), + googleAuthConfig.getProxyPort())); + break; + } + case SOCKS: + Preconditions.checkArgument(!Strings.isNullOrEmpty(googleAuthConfig.getProxyHost())); + proxy = new Proxy(Proxy.Type.HTTP, + new InetSocketAddress(googleAuthConfig.getProxyHost(), + googleAuthConfig.getProxyPort())); + break; + } + } + this.transport = transportBuilder.setProxy(proxy) + .build(); + this.authorizationCodeFlow = new GoogleAuthorizationCodeFlow.Builder( + transport, + new JacksonFactory(), + googleAuthConfig.getClientId(), + googleAuthConfig.getClientSecret(), + ImmutableSet.of("https://www.googleapis.com/auth/userinfo.email")) + .build(); + this.redirectionUrl = (googleAuthConfig.isSecureEndpoint() + ? "https" + : "http") + + "://" + + googleAuthConfig.getServer() + + CALLBACK_PATH; + this.googleAuthConfig = googleAuthConfig; + this.mapper = mapper; + this.credentialsStorage = credentialsStorage; + } + + @Override + public AuthType type() { + return AuthType.OAUTH_GOOGLE; + } + + @Override + public String redirectionURL(String sessionId) { + final String url = authorizationCodeFlow.newAuthorizationUrl() + .setState(sessionId) + .setRedirectUri(this.redirectionUrl) +// .setRedirectUri("http://localhost:8080/auth/google") + .build(); + return !Strings.isNullOrEmpty(googleAuthConfig.getLoginDomain()) + ? (url + "&hd=" + googleAuthConfig.getLoginDomain()) + : url; + } + + @Override + public Optional login(String authToken, String sessionId) { + if (Strings.isNullOrEmpty(authToken)) { + return Optional.empty(); + } + final GoogleAuthorizationCodeTokenRequest googleAuthorizationCodeTokenRequest + = authorizationCodeFlow.newTokenRequest(authToken); + final String email; + try { + final GoogleTokenResponse tokenResponse = googleAuthorizationCodeTokenRequest + .setRedirectUri(this.redirectionUrl) + .execute(); + final Credential credential = authorizationCodeFlow.createAndStoreCredential(tokenResponse, null); + final HttpRequestFactory requestFactory = transport.createRequestFactory(credential); + // Make an authenticated request + final GenericUrl url = new GenericUrl("https://www.googleapis.com/oauth2/v1/userinfo"); + final HttpRequest request = requestFactory.buildGetRequest(url); + request.getHeaders().setContentType("application/json"); + final String jsonIdentity = request.execute().parseAsString(); + log.debug("Identity: {}", jsonIdentity); + email = mapper.readTree(jsonIdentity).get("email").asText(); + } + catch (IOException e) { + log.error("Error logging in using google:", e); + return Optional.empty(); + } + val user = credentialsStorage.getUser(email) + .orElse(null); + if (null == user) { + log.warn("No authorized user found for email: {}", email); + return Optional.empty(); + } + final Duration sessionDuration = AuthUtils.sessionDuration(authConfig); + return credentialsStorage.provisionToken(user.getId(), + sessionId, + TokenType.DYNAMIC, + new Date(new Date().getTime() + sessionDuration.toMilliseconds())); + } + + @Override + public boolean isPreregistrationRequired() { + return false; + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProviderConfig.java new file mode 100644 index 000000000..ba28df58d --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProviderConfig.java @@ -0,0 +1,75 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.flipkart.foxtrot.server.auth.authprovider.AuthConfigVisitor; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.validation.constraints.NotNull; +import java.net.Proxy; + +/** + * + */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class GoogleAuthProviderConfig extends AuthProviderConfig { + @NotEmpty + @NotNull + private String clientId; + + @NotEmpty + @NotNull + private String clientSecret; + + private String loginDomain; + + @NotNull + @NotEmpty + private String server; + + @NotNull + private boolean secureEndpoint; + + private Proxy.Type proxyType; + + private String proxyHost; + + private int proxyPort; + + @Builder + public GoogleAuthProviderConfig( + boolean enabled, + String clientId, + String clientSecret, + String loginDomain, + String server, + boolean secureEndpoint, + Proxy.Type proxyType, + String proxyHost, + int proxyPort) { + super(AuthType.OAUTH_GOOGLE, enabled); + this.clientId = clientId; + this.clientSecret = clientSecret; + this.loginDomain = loginDomain; + this.server = server; + this.secureEndpoint = secureEndpoint; + this.proxyType = proxyType; + this.proxyHost = proxyHost; + this.proxyPort = proxyPort; + } + + public GoogleAuthProviderConfig() { + super(AuthType.OAUTH_GOOGLE); + } + + @Override + public T accept(AuthConfigVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java index 555664b82..0e7889db0 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java @@ -19,4 +19,5 @@ public class CreateUserRequest { @NotEmpty Set roles; Set tables; + boolean isSystem; } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java new file mode 100644 index 000000000..125555f91 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java @@ -0,0 +1,84 @@ +package com.flipkart.foxtrot.server.auth.sessionstore; + +import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; +import com.google.common.base.Strings; +import com.hazelcast.config.InMemoryFormat; +import com.hazelcast.config.MapConfig; +import com.hazelcast.config.NearCacheConfig; +import com.hazelcast.core.IMap; +import io.dropwizard.lifecycle.Managed; +import ru.vyarus.dropwizard.guice.module.installer.order.Order; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +/** + * + */ +@Singleton +@Order(50) +public class DistributedSessionDataStore implements SessionDataStore, Managed { + private static final String MAP_NAME = "REFERER_CACHE"; + private final HazelcastConnection hazelcastConnection; + + private IMap store; + + @Inject + public DistributedSessionDataStore(HazelcastConnection hazelcastConnection) { + this.hazelcastConnection = hazelcastConnection; + this.hazelcastConnection.getHazelcastConfig() + .getMapConfigs() + .put(MAP_NAME, mapConfig()); + } + + + @Override + public void put(String sessionId, Object data) { + if(Strings.isNullOrEmpty(sessionId) || null == data) { + return; + } + store.put(sessionId, data); + } + + @Override + public Optional get(String sessionId) { + if(Strings.isNullOrEmpty(sessionId)) { + return Optional.empty(); + } + return Optional.ofNullable(store.get(sessionId)); + } + + @Override + public void delete(String sessionId) { + if(Strings.isNullOrEmpty(sessionId)) { + return; + } + store.delete(sessionId); + } + + @Override + public void start() throws Exception { + this.store = this.hazelcastConnection.getHazelcast() + .getMap(MAP_NAME); + } + + @Override + public void stop() throws Exception { + //Do nothing + } + + private MapConfig mapConfig() { + MapConfig mapConfig = new MapConfig(); + mapConfig.setReadBackupData(true); + mapConfig.setInMemoryFormat(InMemoryFormat.BINARY); + mapConfig.setTimeToLiveSeconds(300); + mapConfig.setBackupCount(0); + + NearCacheConfig nearCacheConfig = new NearCacheConfig(); + nearCacheConfig.setTimeToLiveSeconds(300); + nearCacheConfig.setInvalidateOnChange(true); + mapConfig.setNearCacheConfig(nearCacheConfig); + return mapConfig; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/SessionDataStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/SessionDataStore.java new file mode 100644 index 000000000..1c8e58c4f --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/SessionDataStore.java @@ -0,0 +1,12 @@ +package com.flipkart.foxtrot.server.auth.sessionstore; + +import java.util.Optional; + +/** + * + */ +public interface SessionDataStore { + void put(String sessionId, Object data); + Optional get(String sessionId); + void delete(String sessionId); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java index 748167e5f..b1d232d6c 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java @@ -23,6 +23,7 @@ import com.flipkart.foxtrot.core.datastore.impl.hbase.HbaseConfig; import com.flipkart.foxtrot.core.email.EmailConfig; import com.flipkart.foxtrot.core.jobs.optimization.EsIndexOptimizationConfig; +import com.flipkart.foxtrot.server.jobs.sessioncleanup.SessionCleanupConfig; import com.flipkart.foxtrot.core.querystore.impl.CacheConfig; import com.flipkart.foxtrot.core.querystore.impl.ClusterConfig; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConfig; @@ -64,8 +65,13 @@ public class FoxtrotServerConfiguration extends Configuration { private CardinalityConfig cardinality; @Valid private EsIndexOptimizationConfig esIndexOptimizationConfig; + + @Valid + private SessionCleanupConfig sessionCleanupConfig; + @Valid private ConsoleHistoryConfig consoleHistoryConfig; + private EmailConfig emailConfig; private CacheConfig cacheConfig; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java index 7b764f44b..f83a70122 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java @@ -36,15 +36,21 @@ import com.flipkart.foxtrot.core.table.impl.DistributedTableMetadataManager; import com.flipkart.foxtrot.core.table.impl.FoxtrotTableManager; import com.flipkart.foxtrot.server.auth.*; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.sessionstore.DistributedSessionDataStore; +import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; import com.flipkart.foxtrot.server.console.ConsolePersistence; import com.flipkart.foxtrot.server.console.ElasticsearchConsolePersistence; import com.flipkart.foxtrot.server.jobs.consolehistory.ConsoleHistoryConfig; +import com.flipkart.foxtrot.server.jobs.sessioncleanup.SessionCleanupConfig; import com.flipkart.foxtrot.sql.fqlstore.FqlStoreService; import com.flipkart.foxtrot.sql.fqlstore.FqlStoreServiceImpl; import com.foxtrot.flipkart.translator.config.SegregationConfiguration; -import com.google.common.cache.CacheBuilderSpec; import com.foxtrot.flipkart.translator.config.TranslatorConfig; +import com.google.common.cache.CacheBuilderSpec; import com.google.common.collect.ImmutableList; import com.google.inject.AbstractModule; import com.google.inject.Provides; @@ -67,7 +73,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; - import java.util.Collections; import java.util.List; import java.util.Objects; @@ -108,6 +113,10 @@ protected void configure() { .to(ESAuthStore.class); bind(new TypeLiteral>() {}) .to(RoleAuthorizer.class); + bind(AuthProvider.class) + .to(GoogleAuthProvider.class); + bind(SessionDataStore.class) + .to(DistributedSessionDataStore.class); } @Provides @@ -164,6 +173,14 @@ public ConsoleHistoryConfig consoleHistoryConfig(FoxtrotServerConfiguration conf : configuration.getConsoleHistoryConfig(); } + @Provides + @Singleton + public SessionCleanupConfig sessionCleanupConfig(FoxtrotServerConfiguration configuration) { + return null == configuration.getSessionCleanupConfig() + ? new SessionCleanupConfig() + : configuration.getSessionCleanupConfig(); + } + @Provides @Singleton public CacheConfig cacheConfig(FoxtrotServerConfiguration configuration) { @@ -245,6 +262,12 @@ public AuthConfig authConfig(FoxtrotServerConfiguration serverConfiguration) { return serverConfiguration.getAuth(); } + @Provides + @Singleton + public GoogleAuthProviderConfig googleAuthProviderConfig(FoxtrotServerConfiguration configuration) { + return (GoogleAuthProviderConfig)configuration.getAuth().getProvider(); + } + @Provides @Singleton public JwtConsumer provideJwtConsumer(AuthConfig config) { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/ExpiredSessionsCleaner.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/ExpiredSessionsCleaner.java new file mode 100644 index 000000000..03f8169b5 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/ExpiredSessionsCleaner.java @@ -0,0 +1,55 @@ +package com.flipkart.foxtrot.server.jobs.sessioncleanup; + +import com.flipkart.foxtrot.core.jobs.BaseJobManager; +import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.flipkart.foxtrot.server.utils.AuthUtils; +import lombok.extern.slf4j.Slf4j; +import net.javacrumbs.shedlock.core.LockConfiguration; +import net.javacrumbs.shedlock.core.LockingTaskExecutor; +import ru.vyarus.dropwizard.guice.module.installer.order.Order; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.time.Instant; +import java.util.Date; +import java.util.concurrent.ScheduledExecutorService; + +/** + * + */ +@Singleton +@Order(55) +@Slf4j +public class ExpiredSessionsCleaner extends BaseJobManager { + private final SessionCleanupConfig sessionCleanupConfig; + private final Provider authStore; + private final AuthConfig authConfig; + + @Inject + public ExpiredSessionsCleaner( + SessionCleanupConfig sessionCleanupConfig, + ScheduledExecutorService scheduledExecutorService, + HazelcastConnection hazelcastConnection, + Provider authStore, + AuthConfig authConfig) { + super(sessionCleanupConfig, scheduledExecutorService, hazelcastConnection); + this.sessionCleanupConfig = sessionCleanupConfig; + this.authStore = authStore; + this.authConfig = authConfig; + } + + @Override + protected void runImpl(LockingTaskExecutor executor, Instant lockAtMostUntil) { + executor.executeWithLock(() -> { + try { + authStore.get() + .deleteExpiredTokens(new Date(), AuthUtils.sessionDuration(authConfig)); + } catch (Exception e) { + log.error("Session cleanup failed: ", e); + } + }, new LockConfiguration(sessionCleanupConfig.getJobName(), lockAtMostUntil)); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/SessionCleanupConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/SessionCleanupConfig.java new file mode 100644 index 000000000..6ec775202 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/SessionCleanupConfig.java @@ -0,0 +1,15 @@ +package com.flipkart.foxtrot.server.jobs.sessioncleanup; + +import com.flipkart.foxtrot.core.jobs.BaseJobConfig; + +/** + * + */ +public class SessionCleanupConfig extends BaseJobConfig { + private static final String JOB_NAME = "ExpiredSessionCleaner"; + + @Override + public String getJobName() { + return JOB_NAME; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java new file mode 100644 index 000000000..b936fa4a0 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java @@ -0,0 +1,25 @@ +package com.flipkart.foxtrot.server.providers.exception; + +import com.flipkart.foxtrot.server.auth.JWTAuthenticationFailure; +import lombok.extern.slf4j.Slf4j; + +import javax.inject.Singleton; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; +import java.net.URI; + +/** + * + */ +@Provider +@Singleton +@Slf4j +public class AuthenticationExceptionHandler implements ExceptionMapper { + + @Override + public Response toResponse(JWTAuthenticationFailure exception) { + log.error("Authentication failure: {}", exception.getMessage()); + return Response.seeOther(URI.create("/foxtrot/google/login")).build(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java index 35d7bc187..418693d48 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java @@ -54,6 +54,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/analytics") @Singleton +@RolesAllowed(FoxtrotRole.Value.QUERY) public class AnalyticsResource { private final QueryExecutor queryExecutor; @@ -99,7 +100,6 @@ public ActionValidationResponse validateQuery(@Valid final ActionRequest request @Path("/download") @Timed @ApiOperation("downloadAnalytics") - @RolesAllowed(FoxtrotRole.Value.QUERY) public StreamingOutput download(@Valid final ActionRequest actionRequest) { ActionResponse actionResponse = queryExecutor.execute(actionRequest); Flattener flattener = new Flattener(objectMapper, actionRequest, new ArrayList<>()); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AsyncResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AsyncResource.java index d8bee310f..4230334aa 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AsyncResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AsyncResource.java @@ -17,11 +17,13 @@ import com.codahale.metrics.annotation.Timed; import com.flipkart.foxtrot.common.ActionResponse; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.cache.CacheManager; import com.flipkart.foxtrot.core.common.AsyncDataToken; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.constraints.NotNull; @@ -38,6 +40,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/async") @Singleton +@RolesAllowed(FoxtrotRole.Value.QUERY) public class AsyncResource { private CacheManager cacheManager; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java index fb1e82502..e9ece20e9 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java @@ -11,6 +11,7 @@ import lombok.val; import org.hibernate.validator.constraints.NotEmpty; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Provider; import javax.validation.Valid; @@ -27,6 +28,7 @@ @Path("/v1/auth") @Produces(MediaType.APPLICATION_JSON) @Api("Auth related APIs. DO NOT expose these for public access.") +@RolesAllowed(FoxtrotRole.Value.SYSADMIN) public class Auth { private final Provider authProvider; private final AuthConfig authConfig; @@ -42,7 +44,7 @@ public Auth(Provider authProvider, AuthConfig authConfig) { @Consumes(MediaType.APPLICATION_JSON) public Response provisionUser(@NotNull @Valid final CreateUserRequest createUserRequest) { val user = new User(createUserRequest.getId(), createUserRequest.getRoles(), createUserRequest.getTables(), - new Date(), new Date()); + createUserRequest.isSystem(), new Date(), new Date()); return Response.ok(authProvider.get().provisionUser(user)).build(); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/CacheUpdateResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/CacheUpdateResource.java index efd78437a..8afab1f6b 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/CacheUpdateResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/CacheUpdateResource.java @@ -20,6 +20,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.Consumes; @@ -38,6 +39,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/cache/update") @Singleton +@PermitAll public class CacheUpdateResource { private ExecutorService executorService; private TableMetadataManager tableMetadataManager; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java index 900b0c89f..119d5fd1c 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java @@ -24,6 +24,7 @@ import lombok.SneakyThrows; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import javax.annotation.security.PermitAll; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.GET; @@ -40,6 +41,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/clusterhealth") @Singleton +@PermitAll public class ClusterHealthResource { private final QueryStore queryStore; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java index f09528cb7..d88c5fecc 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java @@ -6,6 +6,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.Consumes; @@ -22,6 +23,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/cluster") @Singleton +@PermitAll public class ClusterInfoResource { private ClusterManager clusterManager; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java index 87f916647..2de05c399 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java @@ -33,6 +33,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/consoles") @Singleton +@RolesAllowed({FoxtrotRole.Value.QUERY, FoxtrotRole.Value.CONSOLE}) public class ConsoleResource { private ConsolePersistence consolePersistence; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java index a320074bb..5a2d4f281 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java @@ -16,11 +16,13 @@ package com.flipkart.foxtrot.server.resources; import com.codahale.metrics.annotation.Timed; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.server.console.ConsolePersistence; import com.flipkart.foxtrot.server.console.ConsoleV2; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.*; @@ -32,6 +34,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v2/consoles") @Singleton +@RolesAllowed({FoxtrotRole.Value.QUERY, FoxtrotRole.Value.CONSOLE}) public class ConsoleV2Resource { private ConsolePersistence consolePersistence; @@ -44,6 +47,7 @@ public ConsoleV2Resource(ConsolePersistence consolePersistence) { @POST @Timed @ApiOperation("Save Console") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) public ConsoleV2 save(ConsoleV2 console) { consolePersistence.saveV2(console, true); return console; @@ -61,6 +65,7 @@ public ConsoleV2 get(@PathParam("id") final String id) { @Path("/{id}/delete") @Timed @ApiOperation("Delete Console - via id") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) public void delete(@PathParam("id") final String id) { consolePersistence.deleteV2(id); } @@ -93,6 +98,7 @@ public List getOldVersions(@PathParam("name") final String name) { @Path("/{id}/old/delete") @Timed @ApiOperation("Delete old version console - via id") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) public void deleteOldVersion(@PathParam("id") final String id) { consolePersistence.deleteOldVersion(id); } @@ -101,6 +107,7 @@ public void deleteOldVersion(@PathParam("id") final String id) { @Timed @Path("/{id}/old/set/current") @ApiOperation("Set old version console with id: {id} as current console") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) public void setOldVersionAsCurrent(@PathParam("id") final String id) { consolePersistence.setOldVersionAsCurrent(id); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/GoogleAuth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/GoogleAuth.java new file mode 100644 index 000000000..8ca0e7311 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/GoogleAuth.java @@ -0,0 +1,118 @@ +package com.flipkart.foxtrot.server.resources; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProvider; +import com.flipkart.foxtrot.server.utils.AuthUtils; +import com.google.common.base.Strings; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.http.HttpHeaders; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.*; +import java.net.URI; +import java.util.UUID; + +/** + * + */ +@Path("/google") +@Slf4j +public class GoogleAuth { + + private final AuthConfig authConfig; + private final Provider authProvider; + private final Provider sessionDataStore; + + @Inject + public GoogleAuth( + AuthConfig authConfig, + Provider authProvider, + Provider sessionDataStore) { + this.authConfig = authConfig; + this.authProvider = authProvider; + this.sessionDataStore = sessionDataStore; + } + + @GET + @Path("/login") + public Response login(@CookieParam("redirection") final Cookie cookieReferrer, + @HeaderParam(HttpHeaders.REFERER) final String referrer) { + final String sessionId = UUID.randomUUID().toString(); + final String redirectionURL = authProvider.get() + .redirectionURL(sessionId); + log.info("Redirection uri: {}", redirectionURL); + final String cookieReferrerUrl = null == cookieReferrer ? null : cookieReferrer.getValue(); + val source = Strings.isNullOrEmpty(cookieReferrerUrl) ? referrer : cookieReferrerUrl; + log.info("Call source: {} Referrer: {} Redirection: {}", source, referrer, cookieReferrerUrl); + if(!Strings.isNullOrEmpty(source)) { + sessionDataStore.get() + .put(sessionId, source); + log.info("Saved: {} against session: {}", source, sessionId); + } + return Response.seeOther(URI.create(redirectionURL)) + .cookie( + new NewCookie( + "gauth-state", + sessionId, + GoogleAuthProvider.CALLBACK_PATH, + null, + NewCookie.DEFAULT_VERSION, + null, + NewCookie.DEFAULT_MAX_AGE, + null, + false, + false)) + .build(); + } + + @GET + @Path("/callback") + public Response handleGoogleCallback( + @CookieParam("gauth-state") final Cookie cookieState, + @Context HttpServletRequest requestContext, + @QueryParam("state") final String sessionId, + @QueryParam("code") final String authCode) { + log.info("Request Ctx: {}", requestContext); + if (null == cookieState + || !cookieState.getValue().equals(sessionId)) { + return Response.seeOther(URI.create("/")) + .cookie(new NewCookie(cookieState, null, 0, false)) + .build(); + } + val token = authProvider.get() + .login(authCode, sessionId) + .orElse(null); + if (null == token) { + return Response.seeOther(URI.create("/foxtrot/login/google")).build(); + } + val existingReferrer = sessionDataStore.get() + .get(token.getId()) + .orElse(null); + if(null != existingReferrer) { + sessionDataStore.get() + .delete(token.getId()); + } + log.info("Got: {} against session: {}", existingReferrer, authCode); + final String finalRedirect = Strings.isNullOrEmpty((String)existingReferrer) ? "/" : (String)existingReferrer; + log.info("Will be redirecting to: {}. Existing: {}", finalRedirect, existingReferrer); + return Response.seeOther(URI.create(finalRedirect)) + .cookie(new NewCookie("token", + AuthUtils.createJWT(token, authConfig.getJwt()), + "/", + null, + Cookie.DEFAULT_VERSION, + null, + NewCookie.DEFAULT_MAX_AGE, + null, + false, + true), + new NewCookie(cookieState, null, 0, false)) + .build(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResource.java index 2c30b1ebc..6b4b7b6aa 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResource.java @@ -17,11 +17,14 @@ import com.codahale.metrics.annotation.Timed; import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.table.TableManager; import com.flipkart.foxtrot.core.table.TableMetadataManager; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.*; @@ -36,6 +39,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/tables") @Singleton +@PermitAll public class TableFieldMappingResource { private final TableManager tableManager; @@ -80,6 +84,7 @@ public Response getAllFields(@QueryParam("withCardinality") @DefaultValue("false @Timed @Path("/{name}/fields/update") @ApiOperation("Update Fields") + @RolesAllowed(FoxtrotRole.Value.SYSADMIN) public Response updateEstimation(@PathParam("name") final String table, @QueryParam("time") @DefaultValue("0") long epoch) { tableMetadataManager.updateEstimationData(table, 0 == epoch ? System.currentTimeMillis() : epoch); return Response.ok() diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java index 4937745ec..13f952e0a 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java @@ -17,11 +17,14 @@ import com.codahale.metrics.annotation.Timed; import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.table.TableManager; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.Valid; @@ -34,6 +37,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/tables") @Singleton +@PermitAll public class TableManagerResource { private final TableManager tableManager; @@ -43,16 +47,6 @@ public TableManagerResource(TableManager tableManager) { this.tableManager = tableManager; } - @POST - @Timed - @ApiOperation("Save Table") - public Response save(@Valid final Table table, @QueryParam("forceCreate") @DefaultValue("false") boolean forceCreate) { - table.setName(ElasticsearchUtils.getValidTableName(table.getName())); - tableManager.save(table, forceCreate); - return Response.ok(table) - .build(); - } - @GET @Timed @Path("/{name}") @@ -65,10 +59,32 @@ public Response get(@PathParam("name") String name) { .build(); } + @GET + @Timed + @ApiOperation("Get all Tables") + public Response getAll() { + return Response.ok() + .entity(tableManager.getAll()) + .build(); + } + + @POST + @Timed + @ApiOperation("Save Table") + @RolesAllowed(FoxtrotRole.Value.SYSADMIN) + public Response save(@Valid final Table table, @QueryParam("forceCreate") @DefaultValue("false") boolean forceCreate) { + table.setName(ElasticsearchUtils.getValidTableName(table.getName())); + tableManager.save(table, forceCreate); + return Response.ok(table) + .build(); + } + + @PUT @Timed @Path("/{name}") @ApiOperation("Update Table") + @RolesAllowed(FoxtrotRole.Value.SYSADMIN) public Response get(@PathParam("name") final String name, @Valid final Table table) { table.setName(name); tableManager.update(table); @@ -80,6 +96,7 @@ public Response get(@PathParam("name") final String name, @Valid final Table tab @Timed @Path("/{name}/delete") @ApiOperation("Delete Table") + @RolesAllowed(FoxtrotRole.Value.SYSADMIN) public Response delete(@PathParam("name") String name) { name = ElasticsearchUtils.getValidTableName(name); tableManager.delete(name); @@ -87,12 +104,4 @@ public Response delete(@PathParam("name") String name) { .build(); } - @GET - @Timed - @ApiOperation("Get all Tables") - public Response getAll() { - return Response.ok() - .entity(tableManager.getAll()) - .build(); - } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/UtilResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/UtilResource.java index 658e815ef..9adc7ef55 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/UtilResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/UtilResource.java @@ -16,10 +16,13 @@ package com.flipkart.foxtrot.server.resources; import com.codahale.metrics.annotation.Timed; -import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConfig; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.GET; @@ -31,19 +34,24 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/util") @Singleton -public class UtilResource { - private final FoxtrotServerConfiguration configuration; +@PermitAll public class UtilResource { + private final ElasticsearchConfig elasticsearch; + private final ObjectMapper mapper; @Inject - public UtilResource(FoxtrotServerConfiguration configuration) { - this.configuration = configuration; + public UtilResource(ElasticsearchConfig elasticsearch, ObjectMapper mapper) { + this.elasticsearch = elasticsearch; + this.mapper = mapper; } @GET @Path("/config") @Timed @ApiOperation("Get config") - public FoxtrotServerConfiguration configuration() { - return configuration; + public JsonNode configuration() { + return mapper.createObjectNode() + .set("elasticsearch", mapper.createObjectNode() + .put("tableNamePrefix", elasticsearch.getTableNamePrefix()) + .set("hosts", mapper.valueToTree(elasticsearch.getHosts()))); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java index bcc94282e..eba410bda 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java @@ -1,7 +1,9 @@ package com.flipkart.foxtrot.server.utils; +import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.server.auth.JwtConfig; import com.flipkart.foxtrot.server.auth.Token; +import io.dropwizard.util.Duration; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.SneakyThrows; @@ -38,4 +40,11 @@ public static String createJWT(final Token token, final JwtConfig jwtConfig) { jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.HMAC_SHA512); return jws.getCompactSerialization(); } + + public static Duration sessionDuration(AuthConfig authConfig) { + final Duration dynamicSessionDuration = authConfig.getJwt().getSessionDuration(); + return dynamicSessionDuration != null + ? dynamicSessionDuration + : Duration.days(30); + } } diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/ResourceTestUtils.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/ResourceTestUtils.java new file mode 100644 index 000000000..d62f7366e --- /dev/null +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/ResourceTestUtils.java @@ -0,0 +1,30 @@ +package com.flipkart.foxtrot.server; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.server.auth.DummyAuthFilter; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper; +import io.dropwizard.auth.AuthDynamicFeature; +import io.dropwizard.auth.AuthValueFactoryProvider; +import io.dropwizard.testing.junit.ResourceTestRule; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature; +import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory; + +/** + * + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class ResourceTestUtils { + public static ResourceTestRule.Builder testResourceBuilder(ObjectMapper mapper) { + return ResourceTestRule + .builder() + .setTestContainerFactory(new GrizzlyWebTestContainerFactory()) + .addProvider(new AuthDynamicFeature(new DummyAuthFilter())) + .addProvider(RolesAllowedDynamicFeature.class) + .addProvider(new AuthValueFactoryProvider.Binder<>(UserPrincipal.class)) + .setMapper(mapper) + .addProvider(new FoxtrotExceptionMapper(mapper)); + } +} diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/DummyAuthFilter.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/DummyAuthFilter.java new file mode 100644 index 000000000..38965337a --- /dev/null +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/DummyAuthFilter.java @@ -0,0 +1,45 @@ +package com.flipkart.foxtrot.server.auth; + + +import io.dropwizard.auth.AuthFilter; + +import javax.annotation.Priority; +import javax.ws.rs.Priorities; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.core.SecurityContext; +import java.io.IOException; +import java.security.Principal; + +/** + * + */ +@Priority(Priorities.AUTHENTICATION) +public class DummyAuthFilter extends AuthFilter { + + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + requestContext.setSecurityContext(new SecurityContext() { + + @Override + public Principal getUserPrincipal() { + return UserPrincipal.DEFAULT; + } + + @Override + public boolean isUserInRole(String role) { + return true; + } + + @Override + public boolean isSecure() { + return requestContext.getSecurityContext().isSecure(); + } + + @Override + public String getAuthenticationScheme() { + return SecurityContext.BASIC_AUTH; + } + + }); + } +} diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java index 66217bf1a..e4021739e 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java @@ -3,10 +3,10 @@ import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.auth.User; -import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.table.impl.ElasticsearchTestUtils; import com.google.common.collect.Sets; import io.dropwizard.jackson.Jackson; +import io.dropwizard.util.Duration; import lombok.SneakyThrows; import lombok.val; import org.junit.Assert; @@ -14,6 +14,7 @@ import org.junit.Test; import java.util.Collections; +import java.util.Date; import java.util.EnumSet; import java.util.Objects; @@ -22,12 +23,11 @@ */ public class ESAuthStoreTest { - private ElasticsearchConnection elasticsearchConnection; private AuthStore authStore; @Before public void setup() throws Exception { - elasticsearchConnection = ElasticsearchTestUtils.getConnection(); + val elasticsearchConnection = ElasticsearchTestUtils.getConnection(); TestUtils.ensureIndex(elasticsearchConnection, ESAuthStore.USERS_INDEX); TestUtils.ensureIndex(elasticsearchConnection, ESAuthStore.TOKENS_INDEX); authStore = new ESAuthStore(elasticsearchConnection, Jackson.newObjectMapper()); @@ -40,6 +40,7 @@ public void usersTest() { final User originalUser = new User(TEST_USER, EnumSet.of(FoxtrotRole.INGEST), Collections.emptySet(), + true, null, null); Assert.assertEquals(originalUser, authStore.provisionUser(originalUser).orElse(null)); @@ -64,7 +65,7 @@ public void tokensTest() { .orElse(null)); final User originalUser = new User(TEST_USER, EnumSet.of(FoxtrotRole.INGEST), - Collections.emptySet(), null, null); + Collections.emptySet(), true,null, null); Assert.assertEquals(originalUser, authStore.provisionUser(originalUser).orElse(null)); val token = authStore.provisionToken(TEST_USER, TokenType.STATIC, null) .orElse(null); @@ -78,4 +79,48 @@ public void tokensTest() { Assert.assertNull(authStore.getToken(token.getId()).orElse(null)); Assert.assertTrue(authStore.deleteUser(TEST_USER)); } + + @Test + public void tokensDeleteTest() { + final String TEST_USER = "test_user"; + Assert.assertNull(authStore.provisionToken(TEST_USER, TokenType.STATIC, null) + .orElse(null)); + final User originalUser = new User(TEST_USER, + EnumSet.of(FoxtrotRole.INGEST), + Collections.emptySet(), true,null, null); + Assert.assertEquals(originalUser, authStore.provisionUser(originalUser).orElse(null)); + val token = authStore.provisionToken(TEST_USER, TokenType.STATIC, null) + .orElse(null); + Assert.assertNotNull(token); + Assert.assertEquals(TEST_USER, token.getUserId()); + Assert.assertEquals(token.getId(), + Objects.requireNonNull(authStore.getToken(token.getId()).orElse(null)).getId()); + Assert.assertEquals(token.getId(), + Objects.requireNonNull(authStore.getTokenForUser(TEST_USER).orElse(null)).getId()); + + //Delete user and make sure tokens are deleted along with that + Assert.assertTrue(authStore.deleteUser(TEST_USER)); + Assert.assertNull(authStore.getToken(token.getId()).orElse(null)); + } + + @Test + public void expiredTokensDeleteTest() { + val sessionDuration = Duration.seconds(2); + final String TEST_USER = "test_user"; + final User originalUser = new User(TEST_USER, + EnumSet.of(FoxtrotRole.INGEST), + Collections.emptySet(), true,null, null); + Assert.assertEquals(originalUser, authStore.provisionUser(originalUser).orElse(null)); + final Date testDate = new Date(); + val tokenDate = new Date(testDate.getTime() - sessionDuration.toMilliseconds() - 5_000); + val token = authStore.provisionToken(TEST_USER, TokenType.STATIC, tokenDate) + .orElse(null); + Assert.assertNotNull(token); + val token2 = authStore.provisionToken(TEST_USER, TokenType.STATIC, null).orElse(null); + Assert.assertNotNull(token2); + + Assert.assertTrue(authStore.deleteExpiredTokens(testDate, sessionDuration)); + Assert.assertNull(authStore.getToken(token.getId()).orElse(null)); + Assert.assertNotNull(authStore.getToken(token2.getId()).orElse(null)); + } } \ No newline at end of file diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AnalyticsResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AnalyticsResourceTest.java index d79e91b74..84d99e2a9 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AnalyticsResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AnalyticsResourceTest.java @@ -20,7 +20,7 @@ import com.flipkart.foxtrot.common.group.GroupResponse; import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.common.AsyncDataToken; -import com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper; +import com.flipkart.foxtrot.server.ResourceTestUtils; import io.dropwizard.testing.junit.ResourceTestRule; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.client.RequestOptions; @@ -54,10 +54,8 @@ public AnalyticsResourceTest() throws Exception { getElasticsearchConnection().getClient() .indices() .refresh(new RefreshRequest("*"), RequestOptions.DEFAULT); - resources = ResourceTestRule.builder() - .setMapper(getMapper()) - .addResource(new AnalyticsResource(getQueryExecutor(),objectMapper)) - .addProvider(new FoxtrotExceptionMapper(getMapper())) + resources = ResourceTestUtils.testResourceBuilder(getMapper()) + .addResource(new AnalyticsResource(getQueryExecutor(), objectMapper)) .build(); } @@ -100,8 +98,7 @@ public void testRunSync() throws Exception { }}); Entity serviceUserEntity = Entity.json(groupRequest); - GroupResponse response = resources.client() - .target("/v1/analytics") + GroupResponse response = resources.target("/v1/analytics") .request() .post(serviceUserEntity, GroupResponse.class); assertEquals(expectedResponse, response.getResult()); @@ -115,8 +112,7 @@ public void testRunSyncInvalidTable() throws Exception { try { Entity serviceUserEntity = Entity.json(groupRequest); - resources.client() - .target("/v1/generate/test") + resources.target("/v1/generate/test") .request() .post(serviceUserEntity, GroupResponse.class); fail(); @@ -163,8 +159,7 @@ public void testRunSyncAsync() throws Exception { put("iphone", iPhoneResponse); }}); Entity serviceUserEntity = Entity.json(groupRequest); - AsyncDataToken response = resources.client() - .target("/v1/analytics/async") + AsyncDataToken response = resources.target("/v1/analytics/async") .request() .post(serviceUserEntity, AsyncDataToken.class); await().pollDelay(2000, TimeUnit.MILLISECONDS).until(() -> true); @@ -181,9 +176,9 @@ public void testRunSyncAsyncInvalidTable() throws Exception { GroupResponse expectedResponse = new GroupResponse(); Entity serviceUserEntity = Entity.json(groupRequest); - AsyncDataToken asyncDataToken = resources.client() - .target("/v1/analytics/async") + AsyncDataToken asyncDataToken = resources.target("/v1/analytics/async") .request() + .header("Authorization", "Bearer TOKEN") .post(serviceUserEntity, AsyncDataToken.class); await().pollDelay(2000, TimeUnit.MILLISECONDS).until(() -> true); GroupResponse actualResponse = GroupResponse.class.cast(getCacheManager().getCacheFor(asyncDataToken.getAction()) diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AsyncResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AsyncResourceTest.java index 790363f5b..a700ac409 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AsyncResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AsyncResourceTest.java @@ -20,7 +20,7 @@ import com.flipkart.foxtrot.common.group.GroupResponse; import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.common.AsyncDataToken; -import com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper; +import com.flipkart.foxtrot.server.ResourceTestUtils; import io.dropwizard.testing.junit.ResourceTestRule; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.client.RequestOptions; @@ -51,10 +51,8 @@ public AsyncResourceTest() throws Exception { getElasticsearchConnection().getClient() .indices() .refresh(new RefreshRequest("*"), RequestOptions.DEFAULT); - resources = ResourceTestRule.builder() - .setMapper(getMapper()) + resources = ResourceTestUtils.testResourceBuilder(getMapper()) .addResource(new AsyncResource(getCacheManager())) - .addProvider(new FoxtrotExceptionMapper(getMapper())) .build(); } @@ -98,7 +96,7 @@ public void testGetResponse() throws Exception { AsyncDataToken dataToken = getQueryExecutor().executeAsync(groupRequest); await().pollDelay(1000, TimeUnit.MILLISECONDS).until(() -> true); - GroupResponse groupResponse = resources.client() + GroupResponse groupResponse = resources .target("/v1/async/" + dataToken.getAction() + "/" + dataToken.getKey()) .request() .get(GroupResponse.class); @@ -113,7 +111,7 @@ public void testGetResponseInvalidAction() throws Exception { AsyncDataToken dataToken = getQueryExecutor().executeAsync(groupRequest); await().pollDelay(1000, TimeUnit.MILLISECONDS).until(() -> true); - GroupResponse response = resources.client() + GroupResponse response = resources .target(String.format("/v1/async/distinct/%s", dataToken.getKey())) .request() .get(GroupResponse.class); @@ -129,7 +127,7 @@ public void testGetResponseInvalidKey() throws Exception { AsyncDataToken dataToken = getQueryExecutor().executeAsync(groupRequest); await().pollDelay(1000, TimeUnit.MILLISECONDS).until(() -> true); - GroupResponse response = resources.client() + GroupResponse response = resources .target(String.format("/v1/async/%s/dummy", dataToken.getAction())) .request() .get(GroupResponse.class); @@ -178,7 +176,7 @@ public void testGetResponsePost() throws Exception { Entity asyncDataTokenEntity = Entity.json(dataToken); - GroupResponse response = resources.client() + GroupResponse response = resources .target("/v1/async") .request() .post(asyncDataTokenEntity, GroupResponse.class); @@ -190,7 +188,7 @@ public void testGetResponsePost() throws Exception { public void testGetResponsePostInvalidKey() throws Exception { AsyncDataToken dataToken = new AsyncDataToken("group", null); Entity asyncDataTokenEntity = Entity.json(dataToken); - GroupResponse response = resources.client() + GroupResponse response = resources .target("/v1/async") .request() .post(asyncDataTokenEntity, GroupResponse.class); @@ -202,7 +200,7 @@ public void testGetResponsePostInvalidAction() throws Exception { AsyncDataToken dataToken = new AsyncDataToken(null, UUID.randomUUID() .toString()); Entity asyncDataTokenEntity = Entity.json(dataToken); - Response response = resources.client() + Response response = resources .target("/v1/async") .request() .post(asyncDataTokenEntity); diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/CacheUpdateResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/CacheUpdateResourceTest.java index 9bca80057..9ae5eb5cc 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/CacheUpdateResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/CacheUpdateResourceTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -import com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper; +import com.flipkart.foxtrot.server.ResourceTestUtils; import io.dropwizard.testing.junit.ResourceTestRule; import org.junit.Rule; import org.junit.Test; @@ -33,17 +33,14 @@ public class CacheUpdateResourceTest extends FoxtrotResourceTest { public CacheUpdateResourceTest() throws Exception { super(); ExecutorService executorService = Executors.newFixedThreadPool(1); - resources = ResourceTestRule.builder() + resources = ResourceTestUtils.testResourceBuilder(getMapper()) .addResource(new CacheUpdateResource(executorService, getTableMetadataManager())) - .addProvider(new FoxtrotExceptionMapper(getMapper())) - .setMapper(getMapper()) .build(); } @Test public void testUpdateCache() { - resources.client() - .target("/v1/cache/update/cardinality") + resources.target("/v1/cache/update/cardinality") .request() .post(null); diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/ClusterHealthResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/ClusterHealthResourceTest.java index ecae9ee0a..dbdf15fa7 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/ClusterHealthResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/ClusterHealthResourceTest.java @@ -21,7 +21,7 @@ import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper; +import com.flipkart.foxtrot.server.ResourceTestUtils; import io.dropwizard.testing.junit.ResourceTestRule; import org.junit.Assert; import org.junit.Ignore; @@ -42,16 +42,14 @@ public class ClusterHealthResourceTest extends FoxtrotResourceTest { public ClusterHealthResourceTest() throws Exception { super(); - resources = ResourceTestRule.builder() + resources = ResourceTestUtils.testResourceBuilder(getMapper()) .addResource(new ClusterHealthResource(getQueryStore())) - .addProvider(new FoxtrotExceptionMapper(getMapper())) - .setMapper(getMapper()) .build(); } @Test public void testClusterHealthApi() { - JsonNode response = resources.client() + JsonNode response = resources .target("/v1/clusterhealth") .request() .get(JsonNode.class); @@ -61,7 +59,7 @@ public void testClusterHealthApi() { @Test public void testNodeStats() throws FoxtrotException { - JsonNode response = resources.client() + JsonNode response = resources .target("/v1/clusterhealth/nodestats") .request() .get(JsonNode.class); @@ -86,7 +84,7 @@ public void testIndicesStats() throws FoxtrotException { documents.add(document2); getQueryStore().save(TestUtils.TEST_TABLE_NAME, documents); getElasticsearchConnection().refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); - JsonNode response = resources.client() + JsonNode response = resources .target("/v1/clusterhealth/indicesstats") .request() .get(JsonNode.class); diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java index 95581a9b7..7eaa17b33 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java @@ -20,7 +20,7 @@ import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.exception.FoxtrotExceptions; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper; +import com.flipkart.foxtrot.server.ResourceTestUtils; import com.foxtrot.flipkart.translator.TableTranslator; import com.foxtrot.flipkart.translator.config.SegregationConfiguration; import io.dropwizard.testing.junit.ResourceTestRule; @@ -46,10 +46,8 @@ public class DocumentResourceTest extends FoxtrotResourceTest { @Rule - public ResourceTestRule resources = ResourceTestRule.builder() + public ResourceTestRule resources = ResourceTestUtils.testResourceBuilder(getMapper()) .addResource(new DocumentResource(getQueryStore(), new TableTranslator(new SegregationConfiguration()))) - .addProvider(new FoxtrotExceptionMapper(getMapper())) - .setMapper(objectMapper) .build(); @Test @@ -60,7 +58,7 @@ public void testSaveDocument() throws Exception { .objectNode() .put("hello", "world")); Entity documentEntity = Entity.json(document); - resources.client() + resources .target("/v1/document/" + TestUtils.TEST_TABLE_NAME) .request() .post(documentEntity); @@ -79,7 +77,7 @@ public void testSaveDocumentInternalError() throws Exception { Entity documentEntity = Entity.json(document); doThrow(FoxtrotExceptions.createExecutionException("dummy", new IOException())).when(getQueryStore()) .save(anyString(), Matchers.any()); - Response response = resources.client() + Response response = resources .target("/v1/document/" + TestUtils.TEST_TABLE_NAME) .request() .post(documentEntity); @@ -92,7 +90,7 @@ public void testSaveDocumentNullId() throws Exception { .objectNode() .put("hello", "world")); Entity documentEntity = Entity.json(document); - Response response = resources.client() + Response response = resources .target("/v1/document/" + TestUtils.TEST_TABLE_NAME) .request() .post(documentEntity); @@ -104,7 +102,7 @@ public void testSaveDocumentNullData() throws Exception { Document document = new Document(UUID.randomUUID() .toString(), System.currentTimeMillis(), null); Entity documentEntity = Entity.json(document); - Response response = resources.client() + Response response = resources .target("/v1/document/" + TestUtils.TEST_TABLE_NAME) .request() .post(documentEntity); @@ -113,7 +111,7 @@ public void testSaveDocumentNullData() throws Exception { @Test public void testSaveDocumentUnknownObject() throws Exception { - Response response = resources.client() + Response response = resources .target("/v1/document/" + TestUtils.TEST_TABLE_NAME) .request() .post(Entity.text("hello")); @@ -122,7 +120,7 @@ public void testSaveDocumentUnknownObject() throws Exception { @Test public void testSaveDocumentEmptyJson() throws Exception { - Response response = resources.client() + Response response = resources .target("/v1/document/" + TestUtils.TEST_TABLE_NAME) .request() .post(Entity.json("{}")); @@ -146,7 +144,7 @@ public void testSaveDocuments() throws Exception { documents.add(document1); documents.add(document2); Entity> listEntity = Entity.json(documents); - resources.client() + resources .target(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)) .request() .post(listEntity); @@ -173,7 +171,7 @@ public void testSaveDocumentsInternalError() throws Exception { doThrow(FoxtrotExceptions.createExecutionException("dummy", new IOException())).when(getQueryStore()) .save(anyString(), anyListOf(Document.class)); Entity> listEntity = Entity.json(documents); - Response response = resources.client() + Response response = resources .target(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)) .request() .post(listEntity); @@ -182,7 +180,7 @@ public void testSaveDocumentsInternalError() throws Exception { @Test public void testSaveDocumentsNullDocuments() throws Exception { - Response response = resources.client() + Response response = resources .target(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)) .request() .post(null); @@ -198,7 +196,7 @@ public void testSaveDocumentsNullDocument() throws Exception { .objectNode() .put("d", "d"))); Entity> listEntity = Entity.json(documents); - Response response = resources.client() + Response response = resources .target(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)) .request() .post(listEntity); @@ -212,7 +210,7 @@ public void testSaveDocumentsNullId() throws Exception { .objectNode() .put("d", "d"))); Entity> listEntity = Entity.json(documents); - Response response = resources.client() + Response response = resources .target(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)) .request() .post(listEntity); @@ -225,7 +223,7 @@ public void testSaveDocumentsNullData() throws Exception { documents.add(new Document(UUID.randomUUID() .toString(), System.currentTimeMillis(), null)); Entity> listEntity = Entity.json(documents); - Response response = resources.client() + Response response = resources .target(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)) .request() .post(listEntity); @@ -234,7 +232,7 @@ public void testSaveDocumentsNullData() throws Exception { @Test public void testSaveDocumentsInvalidRequestObject() throws Exception { - Response response = resources.client() + Response response = resources .target(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)) .request() .post(Entity.text("Hello")); @@ -244,7 +242,7 @@ public void testSaveDocumentsInvalidRequestObject() throws Exception { @Test public void testSaveDocumentsEmptyList() throws Exception { Entity> list = Entity.json(Collections.emptyList()); - Response response = resources.client() + Response response = resources .target(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)) .request() .post(list); @@ -260,7 +258,7 @@ public void testGetDocument() throws Exception { .put("D", "data")); getQueryStore().save(TestUtils.TEST_TABLE_NAME, document); getElasticsearchConnection().refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); - Document response = resources.client() + Document response = resources .target(String.format("/v1/document/%s/%s", TestUtils.TEST_TABLE_NAME, id)) .request() .get(Document.class); @@ -272,7 +270,7 @@ public void testGetDocumentMissingId() throws Exception { String id = UUID.randomUUID() .toString(); try { - resources.client() + resources .target(String.format("/v1/document/%s/%s", TestUtils.TEST_TABLE_NAME, id)) .request() .get(Document.class); @@ -290,7 +288,7 @@ public void testGetDocumentInternalError() throws Exception { try { doThrow(FoxtrotExceptions.createExecutionException("dummy", new IOException())).when(getQueryStore()) .get(anyString(), anyString()); - resources.client() + resources .target(String.format("/v1/document/%s/%s", TestUtils.TEST_TABLE_NAME, id)) .request() .get(Document.class); @@ -319,7 +317,7 @@ public void testGetDocuments() throws Exception { documents.add(document2); getQueryStore().save(TestUtils.TEST_TABLE_NAME, documents); getElasticsearchConnection().refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); - String response = resources.client() + String response = resources .target(String.format("/v1/document/%s", TestUtils.TEST_TABLE_NAME)) .queryParam("id", id1) .queryParam("id", id2) @@ -331,7 +329,7 @@ public void testGetDocuments() throws Exception { @Test(expected = BadRequestException.class) public void testGetDocumentsNoIds() throws Exception { - String response = resources.client() + String response = resources .target(String.format("/v1/document/%s", TestUtils.TEST_TABLE_NAME)) .request() .get(String.class); @@ -340,7 +338,7 @@ public void testGetDocumentsNoIds() throws Exception { @Test public void testGetDocumentsMissingIds() throws Exception { try { - resources.client() + resources .target(String.format("/v1/document/%s", TestUtils.TEST_TABLE_NAME)) .queryParam("id", UUID.randomUUID() .toString()) @@ -358,7 +356,7 @@ public void testGetDocumentsInternalError() throws Exception { try { doThrow(FoxtrotExceptions.createExecutionException("dummy", new IOException())).when(getQueryStore()) .getAll(anyString(), anyListOf(String.class)); - resources.client() + resources .target(String.format("/v1/document/%s", TestUtils.TEST_TABLE_NAME)) .queryParam("id", UUID.randomUUID() .toString()) diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/FoxtrotResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/FoxtrotResourceTest.java index 667820e57..50f979b0a 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/FoxtrotResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/FoxtrotResourceTest.java @@ -152,7 +152,7 @@ protected FoxtrotResourceTest() { analyticsLoader.start(); TestUtils.registerActions(analyticsLoader, mapper); } catch (Exception e) { - log.error("Error in intialization", e); + log.error("Error in initialization", e); Assert.fail(); } ExecutorService executorService = Executors.newFixedThreadPool(1); diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResourceTest.java index 42a94243f..2759a7aad 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResourceTest.java @@ -20,11 +20,8 @@ import com.flipkart.foxtrot.common.TableFieldMapping; import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.table.impl.FoxtrotTableManager; -import com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper; +import com.flipkart.foxtrot.server.ResourceTestUtils; import io.dropwizard.testing.junit.ResourceTestRule; -import java.util.concurrent.TimeUnit; -import org.awaitility.Awaitility; -import org.awaitility.Durations; import org.junit.Assert; import org.junit.Ignore; import org.junit.Rule; @@ -37,6 +34,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.awaitility.Awaitility.await; import static org.junit.Assert.*; @@ -57,10 +55,8 @@ public TableFieldMappingResourceTest() throws Exception { super(); tableManager = mock(FoxtrotTableManager.class); when(tableManager.getAll()).thenReturn(Collections.singletonList(TestUtils.TEST_TABLE)); - resources = ResourceTestRule.builder() + resources = ResourceTestUtils.testResourceBuilder(getMapper()) .addResource(new TableFieldMappingResource(tableManager, getTableMetadataManager())) - .setMapper(getMapper()) - .addProvider(new FoxtrotExceptionMapper(getMapper())) .build(); } @@ -121,7 +117,7 @@ public void testGet() throws Exception { .build()); TableFieldMapping tableFieldMapping = new TableFieldMapping(TestUtils.TEST_TABLE_NAME, mappings); - String response = resources.client() + String response = resources .target(String.format("/v1/tables/%s/fields", TestUtils.TEST_TABLE_NAME)) .request() .get(String.class); @@ -135,7 +131,7 @@ public void testGet() throws Exception { @Test public void testGetInvalidTable() throws Exception { try { - resources.client() + resources .target(String.format("/v1/tables/%s/fields", TestUtils.TEST_TABLE_NAME + "-missing")) .request() .head(); @@ -148,7 +144,7 @@ public void testGetInvalidTable() throws Exception { @Test public void testGetTableWithNoDocument() throws Exception { TableFieldMapping request = new TableFieldMapping(TestUtils.TEST_TABLE_NAME, new HashSet<>()); - TableFieldMapping response = resources.client() + TableFieldMapping response = resources .target(String.format("/v1/tables/%s/fields", TestUtils.TEST_TABLE_NAME)) .request() .get(TableFieldMapping.class); @@ -164,7 +160,7 @@ public void getAllFields() throws Exception { getQueryStore().save(TestUtils.TEST_TABLE_NAME, TestUtils.getMappingDocuments(getMapper())); await().pollDelay(500, TimeUnit.MILLISECONDS).until(() -> true); - Map response = resources.client() + Map response = resources .target("/v1/tables/fields") .request() .get(new GenericType>() { diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableManagerResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableManagerResourceTest.java index ede06c41f..d37eccce3 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableManagerResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableManagerResourceTest.java @@ -20,7 +20,7 @@ import com.flipkart.foxtrot.core.exception.FoxtrotExceptions; import com.flipkart.foxtrot.core.table.TableManager; import com.flipkart.foxtrot.core.table.impl.FoxtrotTableManager; -import com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper; +import com.flipkart.foxtrot.server.ResourceTestUtils; import io.dropwizard.testing.junit.ResourceTestRule; import org.apache.commons.httpclient.HttpStatus; import org.junit.Rule; @@ -52,10 +52,8 @@ public TableManagerResourceTest() throws Exception { super(); this.tableManager = new FoxtrotTableManager(getTableMetadataManager(), getQueryStore(), getDataStore()); this.tableManager = spy(tableManager); - resources = ResourceTestRule.builder() + resources = ResourceTestUtils.testResourceBuilder(getMapper()) .addResource(new TableManagerResource(tableManager)) - .addProvider(new FoxtrotExceptionMapper(getMapper())) - .setMapper(getMapper()) .build(); } @@ -71,7 +69,7 @@ public void testSave() throws Exception { .ttl(7) .build(); Entity tableEntity = Entity.json(table); - resources.client() + resources .target("/v1/tables") .request() .post(tableEntity); @@ -87,7 +85,7 @@ public void testSave() throws Exception { @Test public void testSaveNullTable() throws Exception { - Response response = resources.client() + Response response = resources .target("/v1/tables") .request() .post(null); @@ -101,7 +99,7 @@ public void testSaveNullTableName() throws Exception { .ttl(30) .build(); Entity
tableEntity = Entity.json(table); - Response response = resources.client() + Response response = resources .target("/v1/tables") .request() .post(tableEntity); @@ -118,7 +116,7 @@ public void testSaveBackendError() throws Exception { Entity
tableEntity = Entity.json(table); doThrow(FoxtrotExceptions.createExecutionException("dummy", new IOException())).when(tableManager) .save(Matchers.
any()); - Response response = resources.client() + Response response = resources .target("/v1/tables") .request() .post(tableEntity); @@ -134,7 +132,7 @@ public void testSaveIllegalTtl() throws Exception { .ttl(0) .build(); Entity
tableEntity = Entity.json(table); - Response response = resources.client() + Response response = resources .target("/v1/tables") .request() .post(tableEntity); @@ -149,7 +147,7 @@ public void testGet() throws Exception { .initializeTable(any(String.class)); - Table response = resources.client() + Table response = resources .target(String.format("/v1/tables/%s", TEST_TABLE_NAME)) .request() .get(Table.class); @@ -160,7 +158,7 @@ public void testGet() throws Exception { @Test public void testGetMissingTable() throws Exception { - Response response = resources.client() + Response response = resources .target(String.format("/v1/tables/%s", TEST_TABLE_NAME + "_missing")) .request() .get(); From 07d0c48242be5b655074413115ffe722396425ff Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 9 Apr 2020 03:41:12 +0530 Subject: [PATCH 09/37] Refresh --- .../com/flipkart/foxtrot/server/auth/ESAuthStore.java | 6 ++++-- .../flipkart/foxtrot/server/auth/ESAuthStoreTest.java | 11 ++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java index 8100bd549..f897d5246 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -179,7 +179,8 @@ public boolean deleteExpiredTokens(Date date, Duration sessionDuration) { .setDocTypes(TOKEN_TYPE) .setIndicesOptions(Utils.indicesOptions()) .setQuery(QueryBuilders.rangeQuery("expiry") - .lt(oldestValidDate.getTime())), RequestOptions.DEFAULT) + .lt(oldestValidDate.getTime())) + .setRefresh(true), RequestOptions.DEFAULT) .getDeleted(); log.info("Deleted {} expired tokens", deletedCount); return true; @@ -204,7 +205,8 @@ public long deleteTokensForUser(String userId) { return connection.getClient() .deleteByQuery(new DeleteByQueryRequest(TOKENS_INDEX) .setDocTypes(TOKEN_TYPE) - .setQuery(QueryBuilders.termQuery("userId", userId)), + .setQuery(QueryBuilders.termQuery("userId", userId)) + .setRefresh(true), RequestOptions.DEFAULT) .getDeleted(); } diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java index e4021739e..096f8d9ef 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/auth/ESAuthStoreTest.java @@ -3,12 +3,14 @@ import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.table.impl.ElasticsearchTestUtils; import com.google.common.collect.Sets; import io.dropwizard.jackson.Jackson; import io.dropwizard.util.Duration; import lombok.SneakyThrows; import lombok.val; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -24,15 +26,21 @@ public class ESAuthStoreTest { private AuthStore authStore; + private ElasticsearchConnection elasticsearchConnection; @Before public void setup() throws Exception { - val elasticsearchConnection = ElasticsearchTestUtils.getConnection(); + elasticsearchConnection = ElasticsearchTestUtils.getConnection(); TestUtils.ensureIndex(elasticsearchConnection, ESAuthStore.USERS_INDEX); TestUtils.ensureIndex(elasticsearchConnection, ESAuthStore.TOKENS_INDEX); authStore = new ESAuthStore(elasticsearchConnection, Jackson.newObjectMapper()); } + @After + public void teardown() { + ElasticsearchTestUtils.cleanupIndices(elasticsearchConnection); + } + @Test @SneakyThrows public void usersTest() { @@ -122,5 +130,6 @@ public void expiredTokensDeleteTest() { Assert.assertTrue(authStore.deleteExpiredTokens(testDate, sessionDuration)); Assert.assertNull(authStore.getToken(token.getId()).orElse(null)); Assert.assertNotNull(authStore.getToken(token2.getId()).orElse(null)); + Assert.assertTrue(authStore.deleteUser(TEST_USER)); } } \ No newline at end of file From 75ed55810653017cee0af7c990b847c1f1c16ab4 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 9 Apr 2020 11:40:49 +0530 Subject: [PATCH 10/37] Removed useless breaking dependency --- foxtrot-server/pom.xml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 5179c03fa..9f1d8578d 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -57,20 +57,11 @@ + - com.github.toastshaman - dropwizard-auth-jwt - ${dropwizard.jwt.auth.version} - - - io.dropwizard - dropwizard-bom - - - io.dropwizard - dropwizard-auth - - + org.bitbucket.b_c + jose4j + 0.7.0 From b5975b326fba368d04b091d2207b60675b862f29 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 9 Apr 2020 21:47:10 +0530 Subject: [PATCH 11/37] Minor cleanup --- foxtrot-server/pom.xml | 4 ++-- .../flipkart/foxtrot/server/auth/JwtAuthFilter.java | 3 +-- .../flipkart/foxtrot/server/resources/GoogleAuth.java | 10 +++++----- .../src/main/resources/console/echo/browse-events.htm | 2 ++ .../src/main/resources/console/echo/fql/index.htm | 1 + 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 9f1d8578d..0a477ecc6 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -12,7 +12,7 @@ foxtrot-server - 1.2.2-0 + 0.7.0 @@ -61,7 +61,7 @@ org.bitbucket.b_c jose4j - 0.7.0 + ${jose4j.version} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java index af26759b6..ac308d4af 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java @@ -45,7 +45,7 @@ private JwtAuthFilter( @Override public void filter(final ContainerRequestContext requestContext) throws IOException { - LOGGER.info("AUTH Called"); + LOGGER.debug("AUTH Called"); if(!authConfig.isEnabled()) { updateContext(requestContext, UserPrincipal.DEFAULT); return; @@ -70,7 +70,6 @@ public void filter(final ContainerRequestContext requestContext) throws IOExcept } throw new JWTAuthenticationFailure(); -// throw new WebApplicationException(unauthorizedHandler.buildResponse(prefix, realm)); } private void updateContext(ContainerRequestContext requestContext, UserPrincipal principal) { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/GoogleAuth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/GoogleAuth.java index 8ca0e7311..e142d16c5 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/GoogleAuth.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/GoogleAuth.java @@ -46,14 +46,14 @@ public Response login(@CookieParam("redirection") final Cookie cookieReferrer, final String sessionId = UUID.randomUUID().toString(); final String redirectionURL = authProvider.get() .redirectionURL(sessionId); - log.info("Redirection uri: {}", redirectionURL); + log.debug("Redirection uri: {}", redirectionURL); final String cookieReferrerUrl = null == cookieReferrer ? null : cookieReferrer.getValue(); val source = Strings.isNullOrEmpty(cookieReferrerUrl) ? referrer : cookieReferrerUrl; - log.info("Call source: {} Referrer: {} Redirection: {}", source, referrer, cookieReferrerUrl); + log.debug("Call source: {} Referrer: {} Redirection: {}", source, referrer, cookieReferrerUrl); if(!Strings.isNullOrEmpty(source)) { sessionDataStore.get() .put(sessionId, source); - log.info("Saved: {} against session: {}", source, sessionId); + log.debug("Saved: {} against session: {}", source, sessionId); } return Response.seeOther(URI.create(redirectionURL)) .cookie( @@ -98,9 +98,9 @@ public Response handleGoogleCallback( sessionDataStore.get() .delete(token.getId()); } - log.info("Got: {} against session: {}", existingReferrer, authCode); + log.debug("Got: {} against session: {}", existingReferrer, authCode); final String finalRedirect = Strings.isNullOrEmpty((String)existingReferrer) ? "/" : (String)existingReferrer; - log.info("Will be redirecting to: {}. Existing: {}", finalRedirect, existingReferrer); + log.debug("Will be redirecting to: {}. Existing: {}", finalRedirect, existingReferrer); return Response.seeOther(URI.create(finalRedirect)) .cookie(new NewCookie("token", AuthUtils.createJWT(token, authConfig.getJwt()), diff --git a/foxtrot-server/src/main/resources/console/echo/browse-events.htm b/foxtrot-server/src/main/resources/console/echo/browse-events.htm index 3aaec5fdd..ca35ed910 100644 --- a/foxtrot-server/src/main/resources/console/echo/browse-events.htm +++ b/foxtrot-server/src/main/resources/console/echo/browse-events.htm @@ -61,6 +61,8 @@

  • Browse Events
    • +
  • Cluster
    • +
  • FQL
  • Create New Dashboard
    • diff --git a/foxtrot-server/src/main/resources/console/echo/fql/index.htm b/foxtrot-server/src/main/resources/console/echo/fql/index.htm index a8839a722..0951accf6 100644 --- a/foxtrot-server/src/main/resources/console/echo/fql/index.htm +++ b/foxtrot-server/src/main/resources/console/echo/fql/index.htm @@ -87,6 +87,7 @@

    • Browse Events
      • +
    • Cluster
    • FQL
    • Create New Dashboard From aae17b101849d78ce9adbf1ac088f14ce09a8bbf Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Fri, 10 Apr 2020 17:37:46 +0530 Subject: [PATCH 12/37] Ability to set url scheme for swagger --- config/docker.yml | 4 +++- docker-compose-auth.yml | 2 ++ docker-compose.yml | 2 ++ .../foxtrot/server/FoxtrotServer.java | 23 ++++++++++--------- .../config/FoxtrotServerConfiguration.java | 6 ++++- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/config/docker.yml b/config/docker.yml index 4df77ae35..3ea3c3d58 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -6,6 +6,8 @@ server: - type: http port: 17001 +swaggerScheme: ${SWAGGER_SCHEME} + elasticsearch: hosts: - ${ELASTICSEARCH_HOST} @@ -76,7 +78,7 @@ auth: enabled: ${AUTH_ENABLED} jwt: issuerId: foxtrot-server - privateKey: "6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" + privateKey: ${PRIVATE_HEY} sessionDuration: 1h provider: diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml index 78e3ad808..3073731d8 100644 --- a/docker-compose-auth.yml +++ b/docker-compose-auth.yml @@ -62,3 +62,5 @@ services: - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} - GOOGLE_SECURE_ENDPOINT=false - DISCOVERY_TYPE=foxtrot_simple + - SWAGGER_SCHEME="http" + - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" diff --git a/docker-compose.yml b/docker-compose.yml index 24cb00c7a..0491e9eb8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,3 +62,5 @@ services: - GOOGLE_CLIENT_SECRET=dummy - GOOGLE_SECURE_ENDPOINT=false - DISCOVERY_TYPE=foxtrot_simple + - SWAGGER_SCHEME="http" + - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java index 83990fa98..9f623264f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java @@ -26,6 +26,7 @@ import com.flipkart.foxtrot.core.util.MetricUtil; import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; import com.flipkart.foxtrot.server.di.FoxtrotModule; +import com.google.common.base.Strings; import com.google.inject.Stage; import io.dropwizard.Application; import io.dropwizard.assets.AssetsBundle; @@ -63,11 +64,20 @@ public boolean withOor() { } }); - final SwaggerBundleConfiguration swaggerBundleConfiguration = getSwaggerBundleConfiguration(); - bootstrap.addBundle(new SwaggerBundle() { @Override protected SwaggerBundleConfiguration getSwaggerBundleConfiguration(FoxtrotServerConfiguration configuration) { + final SwaggerBundleConfiguration swaggerBundleConfiguration = new SwaggerBundleConfiguration(); + swaggerBundleConfiguration.setTitle("Foxtrot"); + swaggerBundleConfiguration.setResourcePackage("com.flipkart.foxtrot.server.resources"); + swaggerBundleConfiguration.setUriPrefix("/foxtrot"); + swaggerBundleConfiguration.setDescription("A store abstraction and analytics system for real-time event data."); + if(!Strings.isNullOrEmpty(configuration.getSwaggerHost())) { + swaggerBundleConfiguration.setHost(configuration.getSwaggerHost()); + } + if(!Strings.isNullOrEmpty(configuration.getSwaggerScheme())) { + swaggerBundleConfiguration.setSchemes(new String[] {configuration.getSwaggerScheme()}); + } return swaggerBundleConfiguration; } }); @@ -117,13 +127,4 @@ private void configureObjectMapper(ObjectMapper objectMapper) { objectMapper.registerSubtypes(new NamedType(KubernetesClusterDiscoveryConfig.class, "foxtrot_kubernetes")); } - private SwaggerBundleConfiguration getSwaggerBundleConfiguration() { - final SwaggerBundleConfiguration swaggerBundleConfiguration = new SwaggerBundleConfiguration(); - swaggerBundleConfiguration.setTitle("Foxtrot"); - swaggerBundleConfiguration.setResourcePackage("com.flipkart.foxtrot.server.resources"); - swaggerBundleConfiguration.setUriPrefix("/foxtrot"); - swaggerBundleConfiguration.setDescription("A store abstraction and analytics system for real-time event data."); - return swaggerBundleConfiguration; - } - } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java index b1d232d6c..afd261ad4 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java @@ -23,12 +23,12 @@ import com.flipkart.foxtrot.core.datastore.impl.hbase.HbaseConfig; import com.flipkart.foxtrot.core.email.EmailConfig; import com.flipkart.foxtrot.core.jobs.optimization.EsIndexOptimizationConfig; -import com.flipkart.foxtrot.server.jobs.sessioncleanup.SessionCleanupConfig; import com.flipkart.foxtrot.core.querystore.impl.CacheConfig; import com.flipkart.foxtrot.core.querystore.impl.ClusterConfig; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConfig; import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.server.jobs.consolehistory.ConsoleHistoryConfig; +import com.flipkart.foxtrot.server.jobs.sessioncleanup.SessionCleanupConfig; import com.foxtrot.flipkart.translator.config.SegregationConfiguration; import com.foxtrot.flipkart.translator.config.TranslatorConfig; import io.dropwizard.Configuration; @@ -86,6 +86,10 @@ public class FoxtrotServerConfiguration extends Configuration { private ElasticsearchTuningConfig elasticsearchTuningConfig; + private String swaggerHost; + + private String swaggerScheme; + @Builder.Default private TranslatorConfig translatorConfig = new TranslatorConfig(); From c6af89fff2724870838d1cb9da5760aee7b73e08 Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Fri, 10 Apr 2020 17:39:05 +0530 Subject: [PATCH 13/37] Merged docker.yml --- config/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/docker.yml b/config/docker.yml index 3ea3c3d58..708b65cdb 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -78,7 +78,7 @@ auth: enabled: ${AUTH_ENABLED} jwt: issuerId: foxtrot-server - privateKey: ${PRIVATE_HEY} + privateKey: ${PRIVATE_KEY} sessionDuration: 1h provider: From b4d7e3be517df7abb42bbec9e89c0159c7018e27 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sat, 11 Apr 2020 14:07:46 +0530 Subject: [PATCH 14/37] Seperated authentication and authorization in two different filters --- config/docker.yml | 2 +- .../server/GlobalAuthCheckingFilter.java | 87 --------- .../server/auth/JwtAuthDynamicFeature.java | 39 ---- .../foxtrot/server/auth/JwtAuthFilter.java | 170 ------------------ .../auth/filter/JwtAuthDynamicFeature.java | 30 ++++ .../server/auth/filter/SessionUser.java | 32 ++++ .../auth/filter/UserAuthenticationFilter.java | 137 ++++++++++++++ .../auth/filter/UserAuthorizationFilter.java | 71 ++++++++ .../server/auth/io/CreateUserRequest.java | 2 +- .../foxtrot/server/di/FoxtrotModule.java | 14 +- 10 files changed, 284 insertions(+), 300 deletions(-) delete mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/GlobalAuthCheckingFilter.java delete mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java delete mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/JwtAuthDynamicFeature.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/SessionUser.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthorizationFilter.java diff --git a/config/docker.yml b/config/docker.yml index 3ea3c3d58..708b65cdb 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -78,7 +78,7 @@ auth: enabled: ${AUTH_ENABLED} jwt: issuerId: foxtrot-server - privateKey: ${PRIVATE_HEY} + privateKey: ${PRIVATE_KEY} sessionDuration: 1h provider: diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/GlobalAuthCheckingFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/GlobalAuthCheckingFilter.java deleted file mode 100644 index b9994efa8..000000000 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/GlobalAuthCheckingFilter.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.flipkart.foxtrot.server; - -import com.flipkart.foxtrot.server.auth.AuthConfig; -import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; -import lombok.extern.slf4j.Slf4j; -import lombok.val; - -import javax.annotation.Priority; -import javax.inject.Inject; -import javax.inject.Provider; -import javax.servlet.*; -import javax.servlet.annotation.WebFilter; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.Priorities; -import javax.ws.rs.core.HttpHeaders; -import java.io.IOException; -import java.util.Arrays; -import java.util.Set; - -/** - * - */ -@Priority(Priorities.AUTHENTICATION) -@WebFilter("/*") -@Slf4j -public class GlobalAuthCheckingFilter implements Filter { - private static final Set WHITELISTED_PATTERNS = ImmutableSet.builder() - .add("/foxtrot/google") - .add("^/foxtrot/auth.*") - .build(); - private final AuthConfig authConfig; - private final Provider sessionDataStore; - - @Inject - public GlobalAuthCheckingFilter( - AuthConfig authConfig, - Provider sessionDataStore) { - this.authConfig = authConfig; - this.sessionDataStore = sessionDataStore; - } - - @Override - public void init(FilterConfig filterConfig) throws ServletException { - - } - - @Override - public void doFilter( - ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { - if(!authConfig.isEnabled()) { - log.trace("Auth disabled"); - chain.doFilter(request, response); - return; - } - HttpServletRequest httpRequest = (HttpServletRequest) request; - HttpServletResponse httpResponse = (HttpServletResponse) response; - final String requestURI = httpRequest.getRequestURI(); - if(WHITELISTED_PATTERNS.stream().anyMatch(requestURI::startsWith)) { - chain.doFilter(request, response); - return; - } - val authHeader = httpRequest.getHeader(HttpHeaders.AUTHORIZATION); - val cookies = httpRequest.getCookies(); - if(!Strings.isNullOrEmpty(authHeader)) { - chain.doFilter(request, response); - return; - } - if(null != cookies && cookies.length != 0 - && Arrays.stream(cookies).anyMatch(cookie -> cookie.getName().equals("token"))) { - chain.doFilter(request, response); - return; - } - val referrer = httpRequest.getHeader(org.apache.http.HttpHeaders.REFERER); - val source = Strings.isNullOrEmpty(referrer) ? requestURI : referrer; - httpResponse.addCookie(new Cookie("redirection", source)); - httpResponse.sendRedirect("/foxtrot/google/login"); - } - - @Override - public void destroy() { - - } -} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java deleted file mode 100644 index 5baf83d9a..000000000 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthDynamicFeature.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.flipkart.foxtrot.server.auth; - -import io.dropwizard.auth.*; -import io.dropwizard.setup.Environment; -import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature; -import org.jose4j.jwt.consumer.JwtConsumer; -import org.jose4j.jwt.consumer.JwtContext; - -import javax.inject.Inject; -import javax.ws.rs.ext.Provider; - -/** - * - */ -@Provider -public class JwtAuthDynamicFeature extends AuthDynamicFeature { - @Inject - public JwtAuthDynamicFeature( - AuthConfig authConfig, - JwtConsumer jwtConsumer, - Authorizer authorizer, - Authenticator authenticator, - Environment environment) { - super(new JwtAuthFilter.Builder() - .setAuthConfig(authConfig) - .setJwtConsumer(jwtConsumer) - .setCookieName("token") - .setPrefix("Bearer") - .setRealm("realm") - .setAuthenticator(authenticator) - .setAuthorizer(authorizer) - .setUnauthorizedHandler(new DefaultUnauthorizedHandler()) - .buildAuthFilter()); - if(null != environment) { - environment.jersey().register(new AuthValueFactoryProvider.Binder<>(UserPrincipal.class)); - environment.jersey().register(RolesAllowedDynamicFeature.class); - } - } -} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java deleted file mode 100644 index ac308d4af..000000000 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtAuthFilter.java +++ /dev/null @@ -1,170 +0,0 @@ -package com.flipkart.foxtrot.server.auth; - -import io.dropwizard.auth.AuthFilter; -import io.dropwizard.auth.AuthenticationException; -import io.dropwizard.auth.Authenticator; -import lombok.val; -import org.jose4j.jwt.consumer.InvalidJwtException; -import org.jose4j.jwt.consumer.JwtConsumer; -import org.jose4j.jwt.consumer.JwtContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Priority; -import javax.ws.rs.InternalServerErrorException; -import javax.ws.rs.Priorities; -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.core.Cookie; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.SecurityContext; -import java.io.IOException; -import java.security.Principal; -import java.util.Map; -import java.util.Optional; - -import static com.google.common.base.Preconditions.checkNotNull; -import static javax.ws.rs.core.HttpHeaders.AUTHORIZATION; - -@Priority(Priorities.AUTHENTICATION) -public class JwtAuthFilter extends AuthFilter { - - private static final Logger LOGGER = LoggerFactory.getLogger(JwtAuthFilter.class); - - private final AuthConfig authConfig; - private final JwtConsumer consumer; - private final String cookieName; - - private JwtAuthFilter( - AuthConfig authConfig, - JwtConsumer consumer, - String cookieName) { - this.authConfig = authConfig; - this.consumer = consumer; - this.cookieName = cookieName; - } - - @Override - public void filter(final ContainerRequestContext requestContext) throws IOException { - LOGGER.debug("AUTH Called"); - if(!authConfig.isEnabled()) { - updateContext(requestContext, UserPrincipal.DEFAULT); - return; - } - final Optional optionalToken = getTokenFromCookieOrHeader(requestContext); - - if (optionalToken.isPresent()) { - try { - val jwtContext = verifyToken(optionalToken.get()); - val principal = authenticator.authenticate(jwtContext); - - if (principal.isPresent()) { - updateContext(requestContext, principal.get()); - return; - } - } catch (InvalidJwtException ex) { - LOGGER.warn("Error decoding credentials: " + ex.getMessage(), ex); - } catch (AuthenticationException ex) { - LOGGER.warn("Error authenticating credentials", ex); - throw new InternalServerErrorException(); - } - } - - throw new JWTAuthenticationFailure(); - } - - private void updateContext(ContainerRequestContext requestContext, UserPrincipal principal) { - requestContext.setSecurityContext(new SecurityContext() { - - @Override - public Principal getUserPrincipal() { - return principal; - } - - @Override - public boolean isUserInRole(String role) { - return authorizer.authorize(principal, role); - } - - @Override - public boolean isSecure() { - return requestContext.getSecurityContext().isSecure(); - } - - @Override - public String getAuthenticationScheme() { - return SecurityContext.BASIC_AUTH; - } - - }); - } - - private JwtContext verifyToken(String rawToken) throws InvalidJwtException { - return consumer.process(rawToken); - } - - private Optional getTokenFromCookieOrHeader(ContainerRequestContext requestContext) { - final Optional headerToken = getTokenFromHeader(requestContext.getHeaders()); - return headerToken.isPresent() ? headerToken : getTokenFromCookie(requestContext); - } - - private Optional getTokenFromHeader(MultivaluedMap headers) { - final String header = headers.getFirst(AUTHORIZATION); - if (header != null) { - int space = header.indexOf(' '); - if (space > 0) { - final String method = header.substring(0, space); - if (prefix.equalsIgnoreCase(method)) { - final String rawToken = header.substring(space + 1); - return Optional.of(rawToken); - } - } - } - - return Optional.empty(); - } - - private Optional getTokenFromCookie(ContainerRequestContext requestContext) { - final Map cookies = requestContext.getCookies(); - - if (cookieName != null && cookies.containsKey(cookieName)) { - final Cookie tokenCookie = cookies.get(cookieName); - final String rawToken = tokenCookie.getValue(); - return Optional.of(rawToken); - } - - return Optional.empty(); - } - - /** - * Builder for {@link JwtAuthFilter}. - *

      An {@link Authenticator} must be provided during the building process.

      - * - */ - public static class Builder extends AuthFilterBuilder { - - private AuthConfig authConfig; - private JwtConsumer consumer; - private String cookieName; - - public Builder setAuthConfig(AuthConfig authConfig) { - this.authConfig = authConfig; - return this; - } - - public Builder setJwtConsumer(JwtConsumer consumer) { - this.consumer = consumer; - return this; - } - - public Builder setCookieName(String cookieName) { - this.cookieName = cookieName; - return this; - } - - @Override - protected JwtAuthFilter newInstance() { - checkNotNull(consumer, "JwtConsumer is not set"); - return new JwtAuthFilter(authConfig, consumer, cookieName); - } - } -} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/JwtAuthDynamicFeature.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/JwtAuthDynamicFeature.java new file mode 100644 index 000000000..2c95782b9 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/JwtAuthDynamicFeature.java @@ -0,0 +1,30 @@ +package com.flipkart.foxtrot.server.auth.filter; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import io.dropwizard.auth.AuthDynamicFeature; +import io.dropwizard.auth.AuthValueFactoryProvider; +import io.dropwizard.auth.Authorizer; +import io.dropwizard.setup.Environment; +import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature; + +import javax.inject.Inject; +import javax.ws.rs.ext.Provider; + +/** + * + */ +@Provider +public class JwtAuthDynamicFeature extends AuthDynamicFeature { + @Inject + public JwtAuthDynamicFeature( + AuthConfig authConfig, + Authorizer authorizer, + Environment environment) { + super(new UserAuthorizationFilter(authConfig, authorizer)); + if(null != environment) { + environment.jersey().register(new AuthValueFactoryProvider.Binder<>(UserPrincipal.class)); + environment.jersey().register(RolesAllowedDynamicFeature.class); + } + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/SessionUser.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/SessionUser.java new file mode 100644 index 000000000..d501c4bd5 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/SessionUser.java @@ -0,0 +1,32 @@ +package com.flipkart.foxtrot.server.auth.filter; + +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import lombok.Builder; +import lombok.Data; +import lombok.val; + +import java.io.Serializable; + +/** + * + */ +@Data +@Builder +public class SessionUser implements Serializable { + private static final long serialVersionUID = -7917711435258380077L; + + private final UserPrincipal user; + + private static ThreadLocal currentUser = new ThreadLocal<>(); + + public static void put(UserPrincipal user) { + currentUser.set(user); + } + + public static UserPrincipal take() { + val user = currentUser.get(); + currentUser.remove(); + return user; + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java new file mode 100644 index 000000000..d6f040de9 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java @@ -0,0 +1,137 @@ +package com.flipkart.foxtrot.server.auth.filter; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import io.dropwizard.auth.AuthenticationException; +import io.dropwizard.auth.Authenticator; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.jose4j.jwt.consumer.InvalidJwtException; +import org.jose4j.jwt.consumer.JwtConsumer; +import org.jose4j.jwt.consumer.JwtContext; + +import javax.annotation.Priority; +import javax.inject.Inject; +import javax.inject.Provider; +import javax.servlet.*; +import javax.servlet.annotation.WebFilter; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Priorities; +import javax.ws.rs.core.HttpHeaders; +import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; +import java.util.Set; + +/** + * This filter validates the token + */ +@Priority(Priorities.AUTHENTICATION) +@WebFilter("/*") +@Slf4j +public class UserAuthenticationFilter implements Filter { + private static final Set WHITELISTED_PATTERNS = ImmutableSet.builder() + .add("/foxtrot/google") + .add("^/foxtrot/auth.*") + .build(); + private final AuthConfig authConfig; + private final Provider sessionDataStore; + private final JwtConsumer consumer; + private final Provider> authenticator; + + @Inject + public UserAuthenticationFilter( + AuthConfig authConfig, + Provider sessionDataStore, + JwtConsumer consumer, + Provider> authenticator) { + this.authConfig = authConfig; + this.sessionDataStore = sessionDataStore; + this.consumer = consumer; + this.authenticator = authenticator; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + + } + + @Override + public void doFilter( + ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + if(!authConfig.isEnabled()) { + log.trace("Auth disabled"); + chain.doFilter(request, response); + return; + } + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + final String requestURI = httpRequest.getRequestURI(); + if(WHITELISTED_PATTERNS.stream().anyMatch(requestURI::startsWith)) { + chain.doFilter(request, response); + return; + } + val jwt = getTokenFromCookieOrHeader(httpRequest).orElse(null); + if(null != jwt) { + try { + final JwtContext context = consumer.process(jwt); + val principal = authenticator.get() + .authenticate(context).orElse(null); + if(null != principal) { + SessionUser.put(principal); + chain.doFilter(request, response); + return; + } + } + catch (InvalidJwtException | AuthenticationException e) { + log.error("Jwt validation failure: ", e); + } + } + val referrer = httpRequest.getHeader(org.apache.http.HttpHeaders.REFERER); + val source = Strings.isNullOrEmpty(referrer) ? requestURI : referrer; + httpResponse.addCookie(new Cookie("redirection", source)); + httpResponse.sendRedirect("/foxtrot/google/login"); + } + + @Override + public void destroy() { + + } + + private Optional getTokenFromCookieOrHeader(HttpServletRequest servletRequest) { + val tokenFromHeader = getTokenFromHeader(servletRequest); + return tokenFromHeader.isPresent() ? tokenFromHeader : getTokenFromCookie(servletRequest); + } + + private Optional getTokenFromHeader(HttpServletRequest servletRequest) { + val header = servletRequest.getHeader(HttpHeaders.AUTHORIZATION); + if (header != null) { + int space = header.indexOf(' '); + if (space > 0) { + final String method = header.substring(0, space); + if ("Bearer ".equalsIgnoreCase(method)) { + final String rawToken = header.substring(space + 1); + return Optional.of(rawToken); + } + } + } + return Optional.empty(); + } + + private Optional getTokenFromCookie(HttpServletRequest request) { + val cookies = request.getCookies(); + if(null != cookies && cookies.length != 0) { + val token = Arrays.stream(cookies).filter(cookie -> cookie.getName().equals("token")).findAny().orElse(null); + if(null != token) { + return Optional.of(token.getValue()); + } + } + return Optional.empty(); + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthorizationFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthorizationFilter.java new file mode 100644 index 000000000..9c16c7991 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthorizationFilter.java @@ -0,0 +1,71 @@ +package com.flipkart.foxtrot.server.auth.filter; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.JWTAuthenticationFailure; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import io.dropwizard.auth.Authorizer; + +import javax.annotation.Priority; +import javax.ws.rs.Priorities; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.core.SecurityContext; +import java.io.IOException; +import java.security.Principal; + +/** + * This filter assigns role to validated user + */ +@Priority(Priorities.AUTHENTICATION) +public class UserAuthorizationFilter implements ContainerRequestFilter { + + private final AuthConfig authConfig; + private final Authorizer authorizer; + + public UserAuthorizationFilter( + AuthConfig authConfig, + Authorizer authorizer) { + this.authConfig = authConfig; + this.authorizer = authorizer; + } + + @Override + public void filter(final ContainerRequestContext requestContext) throws IOException { + if(!authConfig.isEnabled()) { + updateContext(requestContext, UserPrincipal.DEFAULT); + return; + } + UserPrincipal principal = SessionUser.take(); + if(null != principal) { + updateContext(requestContext, principal); + return; + } + throw new JWTAuthenticationFailure(); + } + + private void updateContext(ContainerRequestContext requestContext, UserPrincipal principal) { + requestContext.setSecurityContext(new SecurityContext() { + + @Override + public Principal getUserPrincipal() { + return principal; + } + + @Override + public boolean isUserInRole(String role) { + return authorizer.authorize(principal, role); + } + + @Override + public boolean isSecure() { + return requestContext.getSecurityContext().isSecure(); + } + + @Override + public String getAuthenticationScheme() { + return SecurityContext.BASIC_AUTH; + } + + }); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java index 0e7889db0..61024301f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java @@ -19,5 +19,5 @@ public class CreateUserRequest { @NotEmpty Set roles; Set tables; - boolean isSystem; + boolean system; } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java index f83a70122..9676989b9 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java @@ -58,6 +58,7 @@ import io.dropwizard.auth.Authenticator; import io.dropwizard.auth.Authorizer; import io.dropwizard.auth.CachingAuthenticator; +import io.dropwizard.auth.CachingAuthorizer; import io.dropwizard.server.ServerFactory; import io.dropwizard.setup.Environment; import io.dropwizard.util.Duration; @@ -111,8 +112,6 @@ protected void configure() { bind(new TypeLiteral>() {}).toProvider(HealthcheckListProvider.class); bind(AuthStore.class) .to(ESAuthStore.class); - bind(new TypeLiteral>() {}) - .to(RoleAuthorizer.class); bind(AuthProvider.class) .to(GoogleAuthProvider.class); bind(SessionDataStore.class) @@ -299,6 +298,17 @@ public Authenticator authenticator( CacheBuilderSpec.parse(authConfig.getJwt().getAuthCachePolicy())); } + @Provides + @Singleton + public Authorizer authorizer( + final Environment environment, + final RoleAuthorizer authorizer, + final AuthConfig authConfig) { + return new CachingAuthorizer<>(environment.metrics(), + authorizer, + CacheBuilderSpec.parse(authConfig.getJwt().getAuthCachePolicy())); + } + @Provides @Singleton public ElasticsearchTuningConfig provideElasticsearchTuningConfig(FoxtrotServerConfiguration configuration) { From 9b2aae7f50cccd80b7ee102878e126eb90e8876c Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sat, 11 Apr 2020 15:10:50 +0530 Subject: [PATCH 15/37] Changed default session time --- config/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/docker.yml b/config/docker.yml index 708b65cdb..6e6b76ff2 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -79,7 +79,7 @@ auth: jwt: issuerId: foxtrot-server privateKey: ${PRIVATE_KEY} - sessionDuration: 1h + sessionDuration: 15d provider: type: OAUTH_GOOGLE From 09785c26f89ebe7e8bb20420a2443600254422bc Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sun, 12 Apr 2020 14:58:40 +0530 Subject: [PATCH 16/37] Fixed token parsing --- .../foxtrot/server/auth/filter/UserAuthenticationFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java index d6f040de9..f417728ff 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java @@ -114,7 +114,7 @@ private Optional getTokenFromHeader(HttpServletRequest servletRequest) { int space = header.indexOf(' '); if (space > 0) { final String method = header.substring(0, space); - if ("Bearer ".equalsIgnoreCase(method)) { + if ("Bearer".equalsIgnoreCase(method)) { final String rawToken = header.substring(space + 1); return Optional.of(rawToken); } From 70cd3e645b7c82be1685f62a3e5de4d2f0998b3b Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Thu, 23 Apr 2020 20:12:11 +0530 Subject: [PATCH 17/37] Support for quotes in fql query --- config/local.yml | 28 ++++++++-- ...EventPublisherActionExecutionObserver.java | 3 ++ .../handlers/SlowQueryReporter.java | 2 +- .../com/flipkart/foxtrot/sql/Constants.java | 2 + .../flipkart/foxtrot/sql/QueryTranslator.java | 54 ++++++++++--------- .../flipkart/foxtrot/sql/util/QueryUtils.java | 6 ++- 6 files changed, 63 insertions(+), 32 deletions(-) diff --git a/config/local.yml b/config/local.yml index 0fbf57586..1faf9554b 100755 --- a/config/local.yml +++ b/config/local.yml @@ -8,7 +8,7 @@ server: elasticsearch: hosts: - - localhost + - stg-els601.phonepe.nm2 cluster: foxtrot tableNamePrefix: foxtrot getQueryTimeout: 10000 @@ -18,10 +18,10 @@ elasticsearch: hbase: secure : false tableName: foxtrot - hbaseZookeeperQuorum: localhost:2181 + hbaseZookeeperQuorum: stg-dwhxenial002-stg-cloud005.phonepe.nm2:2181 hbaseZookeeperClientPort: 2181 seggregatedTablePrefix: foxtrot- - hbaseZookeeperZnodeParent: /hbase-unsecure + hbaseZookeeperZnodeParent: /hbase-test cluster: name: foxtrot @@ -49,6 +49,7 @@ elasticsearchTuningConfig: precisionThreshold: 500 aggregationSize: 10000 scrollTimeInSeconds: 120 + documentsLimitAllowed: 10000 esIndexOptimizationConfig: active: true @@ -63,4 +64,23 @@ consoleHistoryConfig: deletionconfig: active: false interval: 86400 - initialdelay: 60 \ No newline at end of file + initialdelay: 60 + +sessionCleanupConfig: + active: true + interval: 86400 + initialDelay: 10 + +auth: + enabled: false + jwt: + issuerId: foxtrot-server + privateKey: ${PRIVATE_KEY} + sessionDuration: 15d + + provider: + type: OAUTH_GOOGLE + clientId: ${GOOGLE_CLIENT_ID} + clientSecret: ${GOOGLE_CLIENT_SECRET} + server: ${GOOGLE_CALLBACK_HOST_PORT} + secureEndpoint: false \ No newline at end of file diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/EventPublisherActionExecutionObserver.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/EventPublisherActionExecutionObserver.java index d493f009d..ec39ec5b8 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/EventPublisherActionExecutionObserver.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/EventPublisherActionExecutionObserver.java @@ -17,6 +17,9 @@ public EventPublisherActionExecutionObserver(InternalEventBus eventBus) { @Override public void postExecution(ActionEvaluationResponse response) { + if (null == response){ + return; + } if(null != response.getException()) { eventBus.publish(new QueryProcessingError(response.getRequest(), response.getException())); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/SlowQueryReporter.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/SlowQueryReporter.java index b03ec13c5..850b715fa 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/SlowQueryReporter.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/SlowQueryReporter.java @@ -12,7 +12,7 @@ public class SlowQueryReporter implements ActionExecutionObserver { @Override public void postExecution(ActionEvaluationResponse response) { - if(null != response.getException() || response.isCached()) { + if(null == response || null != response.getException() || response.isCached()) { return; } if (response.getElapsedTime() > 1000) { diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/Constants.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/Constants.java index 715e3e22c..a27c2454d 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/Constants.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/Constants.java @@ -7,5 +7,7 @@ public class Constants { public static final String SQL_TABLE_REGEX = "[^a-zA-Z0-9\\-_]"; public static final String SQL_FIELD_REGEX = "[^a-zA-Z0-9.\\-_]"; + public static final String REGEX = "^\"+|\"+$"; + public static final String REPLACEMENT = ""; private Constants() {} } diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java index b5461b0b6..fb114f21b 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java @@ -26,6 +26,7 @@ import com.flipkart.foxtrot.sql.query.FqlShowTablesQuery; import com.flipkart.foxtrot.sql.util.QueryUtils; import com.google.common.collect.Lists; +import com.sun.org.apache.regexp.internal.RE; import io.dropwizard.util.Duration; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.*; @@ -43,6 +44,8 @@ import java.io.StringReader; import java.util.List; +import static com.flipkart.foxtrot.sql.Constants.*; + public class QueryTranslator extends SqlElementVisitor { private static final Logger logger = LoggerFactory.getLogger(QueryTranslator.class.getSimpleName()); private static final MetaStatementMatcher metastatementMatcher = new MetaStatementMatcher(); @@ -83,7 +86,7 @@ public void visit(PlainSelect plainSelect) { queryType = FqlQueryType.GROUP; if(groupByItem instanceof Column) { Column column = (Column)groupByItem; - groupBycolumnsList.add(column.getFullyQualifiedName()); + groupBycolumnsList.add(column.getFullyQualifiedName().replaceAll(REGEX, REPLACEMENT)); } } } @@ -141,7 +144,7 @@ public void visit(Select select) { @Override public void visit(Table tableName) { this.tableName = tableName.getName() - .replaceAll(Constants.SQL_TABLE_REGEX, ""); + .replaceAll(SQL_TABLE_REGEX, ""); } @Override @@ -299,7 +302,7 @@ private ResultSort generateResultSort(List orderByElements) { OrderByElement orderByElement = orderByElements.get(0); Column sortColumn = (Column)orderByElement.getExpression(); ResultSort resultSortColumn = new ResultSort(); - resultSortColumn.setField(sortColumn.getFullyQualifiedName()); + resultSortColumn.setField(sortColumn.getFullyQualifiedName().replaceAll(REGEX, REPLACEMENT)); resultSortColumn.setOrder(orderByElement.isAsc() ? ResultSort.Order.asc : ResultSort.Order.desc); logger.info("ResultSort: {}", resultSortColumn); return resultSortColumn; @@ -323,7 +326,7 @@ private List generateColumnSort(List orderItems) { for(OrderByElement orderByElement : orderItems) { Column sortColumn = (Column)orderByElement.getExpression(); ResultSort resultSortColumn = new ResultSort(); - resultSortColumn.setField(sortColumn.getFullyQualifiedName()); + resultSortColumn.setField(sortColumn.getFullyQualifiedName().replaceAll(REGEX, REPLACEMENT)); resultSortColumn.setOrder(orderByElement.isAsc() ? ResultSort.Order.asc : ResultSort.Order.desc); resultSortList.add(resultSortColumn); } @@ -340,7 +343,8 @@ public void visit(SelectExpressionItem selectExpressionItem) { Expression expression = selectExpressionItem.getExpression(); if(expression instanceof Function) { Function function = (Function)expression; - queryType = getType(function.getName()); + String functionName = function.getName().replaceAll(REGEX, REPLACEMENT); + queryType = getType(functionName); switch (queryType) { case TREND: actionRequest = parseTrendFunction(function.getParameters() @@ -370,9 +374,9 @@ public void visit(SelectExpressionItem selectExpressionItem) { } else { if(expression instanceof Parenthesis) { - columnName = ((Column)((Parenthesis)expression).getExpression()).getFullyQualifiedName(); + columnName = ((Column)((Parenthesis)expression).getExpression()).getFullyQualifiedName().replaceAll(REGEX, REPLACEMENT); } else if(expression instanceof Column) { - columnName = ((Column)expression).getFullyQualifiedName(); + columnName = ((Column)expression).getFullyQualifiedName().replaceAll(REGEX, REPLACEMENT); } } } @@ -471,10 +475,10 @@ private ActionRequest parseCountRequest(ExpressionList expressionList, boolean a private String expressionToString(Expression expression) { if(expression instanceof Column) { - return ((Column)expression).getFullyQualifiedName(); + return ((Column)expression).getFullyQualifiedName().replaceAll(REGEX, REPLACEMENT); } if(expression instanceof StringValue) { - return ((StringValue)expression).getValue(); + return ((StringValue)expression).getValue().replaceAll(REGEX, REPLACEMENT); } return null; } @@ -489,7 +493,7 @@ private static final class FilterParser extends SqlElementVisitor { public void visit(EqualsTo equalsTo) { EqualsFilter equalsFilter = new EqualsFilter(); String field = ((Column)equalsTo.getLeftExpression()).getFullyQualifiedName(); - equalsFilter.setField(field.replaceAll(Constants.SQL_FIELD_REGEX, "")); + equalsFilter.setField(field.replaceAll(SQL_FIELD_REGEX, "")); equalsFilter.setValue(getValueFromExpression(equalsTo.getRightExpression())); filters.add(equalsFilter); } @@ -498,7 +502,7 @@ public void visit(EqualsTo equalsTo) { public void visit(NotEqualsTo notEqualsTo) { NotEqualsFilter notEqualsFilter = new NotEqualsFilter(); String field = ((Column)notEqualsTo.getLeftExpression()).getFullyQualifiedName(); - notEqualsFilter.setField(field.replaceAll(Constants.SQL_FIELD_REGEX, "")); + notEqualsFilter.setField(field.replaceAll(SQL_FIELD_REGEX, "")); notEqualsFilter.setValue(getValueFromExpression(notEqualsTo.getRightExpression())); filters.add(notEqualsFilter); } @@ -516,7 +520,7 @@ public void visit(Between between) { BetweenFilter betweenFilter = new BetweenFilter(); ColumnData columnData = setupColumn(between.getLeftExpression()); betweenFilter.setField(columnData.getColumnName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); betweenFilter.setTemporal(columnData.isTemporal()); Number from = getNumbericValue(between.getBetweenExpressionStart()); Number to = getNumbericValue(between.getBetweenExpressionEnd()); @@ -530,7 +534,7 @@ public void visit(GreaterThan greaterThan) { GreaterThanFilter greaterThanFilter = new GreaterThanFilter(); ColumnData columnData = setupColumn(greaterThan.getLeftExpression()); greaterThanFilter.setField(columnData.getColumnName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); greaterThanFilter.setTemporal(columnData.isTemporal()); greaterThanFilter.setValue(getNumbericValue(greaterThan.getRightExpression())); filters.add(greaterThanFilter); @@ -541,7 +545,7 @@ public void visit(GreaterThanEquals greaterThanEquals) { GreaterEqualFilter greaterEqualFilter = new GreaterEqualFilter(); ColumnData columnData = setupColumn(greaterThanEquals.getLeftExpression()); greaterEqualFilter.setField(columnData.getColumnName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); greaterEqualFilter.setTemporal(columnData.isTemporal()); greaterEqualFilter.setValue(getNumbericValue(greaterThanEquals.getRightExpression())); filters.add(greaterEqualFilter); @@ -551,7 +555,7 @@ public void visit(GreaterThanEquals greaterThanEquals) { public void visit(InExpression inExpression) { InFilter inFilter = new InFilter(); inFilter.setField(((Column)inExpression.getLeftExpression()).getFullyQualifiedName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); ItemsList itemsList = inExpression.getRightItemsList(); if(!(itemsList instanceof ExpressionList)) { throw new FqlParsingException("Sub selects not supported"); @@ -565,13 +569,13 @@ public void visit(InExpression inExpression) { if(inExpression.isNot()) { NotInFilter filter = new NotInFilter(); filter.setField(((Column)inExpression.getLeftExpression()).getFullyQualifiedName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); filter.setValues(filterValues); filters.add(filter); } else { InFilter filter = new InFilter(); filter.setField(((Column)inExpression.getLeftExpression()).getFullyQualifiedName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); filter.setValues(filterValues); filters.add(filter); } @@ -585,12 +589,12 @@ public void visit(IsNullExpression isNullExpression) { ExistsFilter existsFilter = new ExistsFilter(); existsFilter.setField(columnData.getColumnName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); filters.add(existsFilter); } else { MissingFilter missingFilter = new MissingFilter(); missingFilter.setField(columnData.getColumnName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); filters.add(missingFilter); } } @@ -601,7 +605,7 @@ public void visit(LikeExpression likeExpression) { ContainsFilter containsFilter = new ContainsFilter(); containsFilter.setValue(getStringValue(likeExpression.getRightExpression())); containsFilter.setField(((Column)likeExpression.getLeftExpression()).getFullyQualifiedName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); filters.add(containsFilter); } @@ -610,7 +614,7 @@ public void visit(MinorThan minorThan) { LessThanFilter lessThanFilter = new LessThanFilter(); ColumnData columnData = setupColumn(minorThan.getLeftExpression()); lessThanFilter.setField(columnData.getColumnName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); lessThanFilter.setTemporal(columnData.isTemporal()); lessThanFilter.setValue(getNumbericValue(minorThan.getRightExpression())); filters.add(lessThanFilter); @@ -622,7 +626,7 @@ public void visit(MinorThanEquals minorThanEquals) { LessEqualFilter lessEqualFilter = new LessEqualFilter(); ColumnData columnData = setupColumn(minorThanEquals.getLeftExpression()); lessEqualFilter.setField(columnData.getColumnName() - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); lessEqualFilter.setTemporal(columnData.isTemporal()); lessEqualFilter.setValue(getNumbericValue(minorThanEquals.getRightExpression())); filters.add(lessEqualFilter); @@ -653,7 +657,7 @@ private LastFilter parseWindowFunction(List expressions) { } if(expressions.size() > 2) { lastFilter.setField(QueryUtils.expressionToString((Expression)expressions.get(2)) - .replaceAll(Constants.SQL_FIELD_REGEX, "")); + .replaceAll(SQL_FIELD_REGEX, "")); } return lastFilter; } @@ -701,12 +705,12 @@ private ColumnData setupColumn(Expression expression) { if(parameters.size() != 1 || !(parameters.get(0) instanceof Column)) { throw new FqlParsingException("temporal function must have a fieldname as parameter"); } - return ColumnData.temporal(((Column)parameters.get(0)).getFullyQualifiedName()); + return ColumnData.temporal(((Column)parameters.get(0)).getFullyQualifiedName().replaceAll(REGEX, REPLACEMENT)); } throw new FqlParsingException("Only the function 'temporal' is supported in where clause"); } if(expression instanceof Column) { - return new ColumnData(((Column)expression).getFullyQualifiedName()); + return new ColumnData(((Column)expression).getFullyQualifiedName().replaceAll(REGEX, REPLACEMENT)); } throw new FqlParsingException("Only the function 'temporal([fieldname)' and fieldname is supported in where clause"); } diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java index d50cd530d..6bec043da 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java @@ -1,5 +1,7 @@ package com.flipkart.foxtrot.sql.util; +import com.flipkart.foxtrot.sql.Constants; +import com.flipkart.foxtrot.sql.QueryTranslator; import net.sf.jsqlparser.expression.DoubleValue; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.LongValue; @@ -12,10 +14,10 @@ private QueryUtils() {} public static String expressionToString(Expression expression) { if(expression instanceof Column) { - return ((Column)expression).getFullyQualifiedName(); + return ((Column)expression).getFullyQualifiedName().replaceAll(Constants.REGEX, Constants.REPLACEMENT); } if(expression instanceof StringValue) { - return ((StringValue)expression).getValue(); + return ((StringValue)expression).getValue().replaceAll(Constants.REGEX, Constants.REPLACEMENT); } return null; } From 937573b03962fd5f382b92cb908990a29c29bf38 Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Thu, 23 Apr 2020 20:17:01 +0530 Subject: [PATCH 18/37] Changes in sql parser --- config/local.yml | 5 +++-- .../server/console/ElasticsearchConsolePersistence.java | 4 ++-- .../flipkart/foxtrot/server/resources/ConsoleV2Resource.java | 2 +- .../main/java/com/flipkart/foxtrot/sql/QueryTranslator.java | 1 - .../main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java | 1 - 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/config/local.yml b/config/local.yml index 1faf9554b..0920b3088 100755 --- a/config/local.yml +++ b/config/local.yml @@ -8,7 +8,7 @@ server: elasticsearch: hosts: - - stg-els601.phonepe.nm2 + - localhost cluster: foxtrot tableNamePrefix: foxtrot getQueryTimeout: 10000 @@ -18,7 +18,7 @@ elasticsearch: hbase: secure : false tableName: foxtrot - hbaseZookeeperQuorum: stg-dwhxenial002-stg-cloud005.phonepe.nm2:2181 + hbaseZookeeperQuorum: localhost:2181 hbaseZookeeperClientPort: 2181 seggregatedTablePrefix: foxtrot- hbaseZookeeperZnodeParent: /hbase-test @@ -48,6 +48,7 @@ cardinality: elasticsearchTuningConfig: precisionThreshold: 500 aggregationSize: 10000 + scrollSize: 100 scrollTimeInSeconds: 120 documentsLimitAllowed: 10000 diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/console/ElasticsearchConsolePersistence.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/console/ElasticsearchConsolePersistence.java index c2b7b75cc..015da8bb4 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/console/ElasticsearchConsolePersistence.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/console/ElasticsearchConsolePersistence.java @@ -54,7 +54,7 @@ @Singleton public class ElasticsearchConsolePersistence implements ConsolePersistence { - public static final String INDEX_HISTORY = "consoles_v2"; + public static final String INDEX_HISTORY = "consoles_history"; public static final String INDEX_V2 = "consoles_v2"; public static final String INDEX = "consoles"; @@ -154,7 +154,7 @@ public void delete(final String id) { @Override public void saveV2(ConsoleV2 console, boolean newConsole) { - //preProcess(console, newConsole); + preProcess(console, newConsole); try { connection.getClient() .index(new IndexRequest(INDEX_V2) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java index 7ff105331..5a2d4f281 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java @@ -82,7 +82,7 @@ public List getList() { @Path("/{id}/old/get") @ApiOperation("get Old Version Console - via id") public ConsoleV2 getOldVersion(@PathParam("id") final String id) { - return consolePersistence.getV2(id); + return consolePersistence.getOldVersion(id); } @GET diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java index fb114f21b..b967a8f34 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java @@ -26,7 +26,6 @@ import com.flipkart.foxtrot.sql.query.FqlShowTablesQuery; import com.flipkart.foxtrot.sql.util.QueryUtils; import com.google.common.collect.Lists; -import com.sun.org.apache.regexp.internal.RE; import io.dropwizard.util.Duration; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.*; diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java index 6bec043da..f541fa494 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/util/QueryUtils.java @@ -1,7 +1,6 @@ package com.flipkart.foxtrot.sql.util; import com.flipkart.foxtrot.sql.Constants; -import com.flipkart.foxtrot.sql.QueryTranslator; import net.sf.jsqlparser.expression.DoubleValue; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.LongValue; From 973ed73abcd6c33a02e367ae07b3728c7479135c Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 7 May 2020 20:23:09 +0530 Subject: [PATCH 19/37] Common filters for multi query --- .../flipkart/foxtrot/core/common/Action.java | 2 +- .../core/querystore/actions/CountAction.java | 11 ++--- .../querystore/actions/DistinctAction.java | 9 ++-- .../core/querystore/actions/FilterAction.java | 22 +++++----- .../core/querystore/actions/GroupAction.java | 8 ++-- .../querystore/actions/HistogramAction.java | 8 ++-- .../querystore/actions/MultiQueryAction.java | 18 ++++++-- .../actions/MultiTimeQueryAction.java | 4 +- .../core/querystore/actions/StatsAction.java | 8 ++-- .../querystore/actions/StatsTrendAction.java | 8 ++-- .../core/querystore/actions/TrendAction.java | 8 ++-- .../core/util/ElasticsearchQueryUtils.java | 17 ++++++++ .../cacheable/DummyCacheableAction.java | 5 ++- .../noncacheable/NonCacheableAction.java | 5 ++- .../actions/MultiQueryActionTest.java | 43 +++++++++++++++++-- .../main/resources/console/js/hostdetails.js | 2 +- 16 files changed, 124 insertions(+), 54 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java index fd5181fe7..30344c41d 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java @@ -144,7 +144,7 @@ private void validateBase(P parameter) { public abstract String getRequestCacheKey(); - public abstract org.elasticsearch.action.ActionRequest getRequestBuilder(P parameter); + public abstract org.elasticsearch.action.ActionRequest getRequestBuilder(P parameter, List extraFilters); public abstract ActionResponse getResponse(org.elasticsearch.action.ActionResponse response, P parameter); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java index ee6c6816c..614a9f92e 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java @@ -13,7 +13,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils.QUERY_SIZE; @@ -97,7 +98,7 @@ public void validateImpl(CountRequest parameter) { @Override public ActionResponse execute(CountRequest parameter) { - SearchRequest request = getRequestBuilder(parameter); + SearchRequest request = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() @@ -112,14 +113,14 @@ public ActionResponse execute(CountRequest parameter) { } @Override - public SearchRequest getRequestBuilder(CountRequest parameter) { + public SearchRequest getRequestBuilder(CountRequest parameter, List extraFilters) { if (parameter.isDistinct()) { try { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(Utils.buildCardinalityAggregation(parameter.getField(), parameter.accept(new CountPrecisionThresholdVisitorAdapter( elasticsearchTuningConfig.getPrecisionThreshold()))))) @@ -135,7 +136,7 @@ public SearchRequest getRequestBuilder(CountRequest parameter) { .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters()))); + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters))); } catch (Exception e) { throw FoxtrotExceptions.queryCreationException(parameter, e); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java index 06028f2eb..d54885497 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java @@ -12,7 +12,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Sets; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -106,7 +107,7 @@ public void validateImpl(DistinctRequest parameter) { public ActionResponse execute(DistinctRequest request) { SearchRequest query; try { - query = getRequestBuilder(request); + query = getRequestBuilder(request, Collections.emptyList()); } catch (Exception e) { throw FoxtrotExceptions.queryCreationException(request, e); @@ -125,12 +126,12 @@ public ActionResponse execute(DistinctRequest request) { } @Override - public SearchRequest getRequestBuilder(DistinctRequest request) { + public SearchRequest getRequestBuilder(DistinctRequest request, List extraFilters) { try { return new SearchRequest(ElasticsearchUtils.getIndices(request.getTable(), request)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() - .query(new ElasticSearchQueryGenerator().genFilter(request.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(request, extraFilters)) .size(QUERY_SIZE) .aggregation(Utils.buildTermsAggregation( request.getNesting(), Sets.newHashSet(), elasticsearchTuningConfig.getAggregationSize())) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java index cb49f25d3..b23915cd6 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java @@ -27,7 +27,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -125,14 +125,14 @@ public void validateImpl(Query parameter) { @Override public ActionResponse execute(Query parameter) { if (parameter.isScrollRequest()) { - return executeScrollRequest(parameter); + return executeScrollRequest(parameter, Collections.emptyList()); } return executeRequest(parameter); } @Override - public SearchRequest getRequestBuilder(Query parameter) { - SearchRequest searchRequest = getSearchRequest(parameter); + public SearchRequest getRequestBuilder(Query parameter, List extraFilters) { + SearchRequest searchRequest = getSearchRequest(parameter, extraFilters); //Adding from here since from isn't supported in scroll request searchRequest.source().from(parameter.getFrom()); return searchRequest; @@ -161,13 +161,13 @@ public ActionResponse getResponse(org.elasticsearch.action.ActionResponse respon } - private SearchRequest getScrollRequestBuilder(Query parameter) { - SearchRequest searchRequest = getSearchRequest(parameter); + private SearchRequest getScrollRequestBuilder(Query parameter, List extraFilters) { + SearchRequest searchRequest = getSearchRequest(parameter, extraFilters); searchRequest.scroll(TimeValue.timeValueSeconds(elasticsearchTuningConfig.getScrollTimeInSeconds())); return searchRequest; } - private SearchRequest getSearchRequest(Query parameter) { + private SearchRequest getSearchRequest(Query parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .types(ElasticsearchUtils.DOCUMENT_TYPE_NAME) @@ -175,13 +175,13 @@ private SearchRequest getSearchRequest(Query parameter) { .source(new SearchSourceBuilder() .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) .size(parameter.getLimit()) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .sort(Utils.storedFieldName(parameter.getSort().getField()), ResultSort.Order.desc == parameter.getSort().getOrder() ? SortOrder.DESC : SortOrder.ASC)); } - private ActionResponse executeScrollRequest(Query parameter) { + private ActionResponse executeScrollRequest(Query parameter, List extraFilters) { List ids = new ArrayList<>(); String scrollId; long totalHits = 0; @@ -199,7 +199,7 @@ private ActionResponse executeScrollRequest(Query parameter) { } } else { - SearchRequest searchRequest = getScrollRequestBuilder(parameter); + SearchRequest searchRequest = getScrollRequestBuilder(parameter, extraFilters); SearchResponse searchResponse = getConnection() .getClient() .search(searchRequest, RequestOptions.DEFAULT); @@ -237,7 +237,7 @@ private QueryResponse getResponse(Query parameter, long totalHits, List } private ActionResponse executeRequest(Query parameter) { - SearchRequest search = getRequestBuilder(parameter); + SearchRequest search = getRequestBuilder(parameter, Collections.emptyList()); try { logger.info("Search: {}", search); SearchResponse response = getConnection() diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java index ab9b54466..3eb9abf8e 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java @@ -42,8 +42,8 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchQueryStore; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.flipkart.foxtrot.core.table.TableMetadataManager; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import lombok.SneakyThrows; @@ -154,7 +154,7 @@ public void validateImpl(GroupRequest parameter) { @Override public ActionResponse execute(GroupRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -167,13 +167,13 @@ public ActionResponse execute(GroupRequest parameter) { } @Override - public SearchRequest getRequestBuilder(GroupRequest parameter) { + public SearchRequest getRequestBuilder(GroupRequest parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(buildAggregation(parameter))); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java index 3e4a47e09..51d464f11 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java @@ -28,7 +28,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import io.dropwizard.util.Duration; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -121,7 +121,7 @@ public void validateImpl(HistogramRequest parameter) { @Override public ActionResponse execute(HistogramRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -134,13 +134,13 @@ public ActionResponse execute(HistogramRequest parameter) { } @Override - public SearchRequest getRequestBuilder(HistogramRequest parameter) { + public SearchRequest getRequestBuilder(HistogramRequest parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(buildAggregation(parameter))); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java index be28eb59a..def05625f 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java @@ -3,6 +3,7 @@ import com.collections.CollectionUtils; import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionResponse; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.common.query.MultiQueryRequest; import com.flipkart.foxtrot.common.query.MultiQueryResponse; import com.flipkart.foxtrot.core.common.Action; @@ -11,8 +12,10 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import lombok.val; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; @@ -81,7 +84,7 @@ public void validateImpl(MultiQueryRequest parameter) { @Override public ActionResponse execute(MultiQueryRequest parameter) { - MultiSearchRequest multiSearchRequestBuilder = getRequestBuilder(parameter); + MultiSearchRequest multiSearchRequestBuilder = getRequestBuilder(parameter, Collections.emptyList()); try { LOGGER.info("Search: {}", multiSearchRequestBuilder); MultiSearchResponse multiSearchResponse = getConnection() @@ -94,10 +97,17 @@ public ActionResponse execute(MultiQueryRequest parameter) { } @Override - public MultiSearchRequest getRequestBuilder(MultiQueryRequest parameter) { + public MultiSearchRequest getRequestBuilder(MultiQueryRequest parameter, List extraFilters) { MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); - + val filterBuilder = ImmutableList.builder(); + if(null != parameter.getFilters()) { + filterBuilder.addAll(parameter.getFilters()); + } + if(null != extraFilters) { + filterBuilder.addAll(extraFilters); + } + val filters = filterBuilder.build(); for(Map.Entry entry : parameter.getRequests() .entrySet()) { @@ -106,7 +116,7 @@ public MultiSearchRequest getRequestBuilder(MultiQueryRequest parameter) { if(null == action) { throw FoxtrotExceptions.queryCreationException(request, null); } - org.elasticsearch.action.ActionRequest requestBuilder = action.getRequestBuilder(request); + org.elasticsearch.action.ActionRequest requestBuilder = action.getRequestBuilder(request, filters); if(requestBuilder instanceof SearchRequest) { multiSearchRequest.add((SearchRequest) requestBuilder); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiTimeQueryAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiTimeQueryAction.java index 0aa91e1f3..8f13de83c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiTimeQueryAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiTimeQueryAction.java @@ -115,8 +115,8 @@ public ActionResponse execute(MultiTimeQueryRequest parameter) { } @Override - public org.elasticsearch.action.ActionRequest getRequestBuilder(MultiTimeQueryRequest parameter) { - return action.getRequestBuilder(multiQueryRequest); + public org.elasticsearch.action.ActionRequest getRequestBuilder(MultiTimeQueryRequest parameter, List extraFilters) { + return action.getRequestBuilder(multiQueryRequest, extraFilters); } @Override diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java index 05fb3e659..f9bfafd1c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java @@ -11,7 +11,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Lists; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -107,7 +107,7 @@ public void validateImpl(StatsRequest parameter) { @Override public ActionResponse execute(StatsRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -120,11 +120,11 @@ public ActionResponse execute(StatsRequest parameter) { } @Override - public SearchRequest getRequestBuilder(StatsRequest parameter) { + public SearchRequest getRequestBuilder(StatsRequest parameter, List extraFilters) { final SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() .size(0) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())); + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)); AbstractAggregationBuilder percentiles = null; final String field = getParameter().getField(); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java index 2e7e96cd6..9b3a85bf1 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java @@ -13,7 +13,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.dropwizard.util.Duration; @@ -119,7 +119,7 @@ public void validateImpl(StatsTrendRequest parameter) { @Override public ActionResponse execute(StatsTrendRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -132,13 +132,13 @@ public ActionResponse execute(StatsTrendRequest parameter) { } @Override - public SearchRequest getRequestBuilder(StatsTrendRequest parameter) { + public SearchRequest getRequestBuilder(StatsTrendRequest parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(0) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(buildAggregation(parameter, parameter.getTable()))); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java index 7ce3d2160..dc620b464 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java @@ -31,7 +31,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.dropwizard.util.Duration; @@ -150,7 +150,7 @@ public void validateImpl(TrendRequest parameter) { @Override public ActionResponse execute(TrendRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -163,13 +163,13 @@ public ActionResponse execute(TrendRequest parameter) { } @Override - public SearchRequest getRequestBuilder(TrendRequest parameter) { + public SearchRequest getRequestBuilder(TrendRequest parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(buildAggregation(parameter))); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/ElasticsearchQueryUtils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/ElasticsearchQueryUtils.java index 49395709f..fa90d7231 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/ElasticsearchQueryUtils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/ElasticsearchQueryUtils.java @@ -16,7 +16,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.common.ActionRequest; +import com.flipkart.foxtrot.common.query.Filter; +import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.google.common.collect.ImmutableList; +import lombok.val; +import org.elasticsearch.index.query.QueryBuilder; +import java.util.List; import java.util.Map; /*** @@ -34,4 +41,14 @@ public static Map toMap(ObjectMapper mapper, return mapper.convertValue(value, new TypeReference>() { }); } + + public static QueryBuilder translateFilter(ActionRequest request, List extraFilters) { + val filters = (null == extraFilters || extraFilters.isEmpty()) + ? request.getFilters() + : ImmutableList.builder() + .addAll(request.getFilters()) + .addAll(extraFilters) + .build(); + return new ElasticSearchQueryGenerator().genFilter(filters); + } } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/cacheable/DummyCacheableAction.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/cacheable/DummyCacheableAction.java index 6b5c1fa6a..14eed3666 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/cacheable/DummyCacheableAction.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/cacheable/DummyCacheableAction.java @@ -16,6 +16,7 @@ package com.flipkart.foxtrot.core.common.cacheable; import com.flipkart.foxtrot.common.ActionResponse; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.core.common.Action; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.flipkart.foxtrot.core.exception.MalformedQueryException; @@ -24,6 +25,8 @@ import com.google.common.annotations.VisibleForTesting; import org.elasticsearch.action.ActionRequest; +import java.util.List; + /** * Created by rishabh.goyal on 02/05/14. */ @@ -57,7 +60,7 @@ public void validateImpl(DummyCacheableActionRequest parameter) throws Malformed } @Override - public ActionRequest getRequestBuilder(DummyCacheableActionRequest parameter) throws FoxtrotException { + public ActionRequest getRequestBuilder(DummyCacheableActionRequest parameter, List extraFilters) throws FoxtrotException { return null; } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/noncacheable/NonCacheableAction.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/noncacheable/NonCacheableAction.java index 44de84424..690f3feea 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/noncacheable/NonCacheableAction.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/noncacheable/NonCacheableAction.java @@ -16,6 +16,7 @@ package com.flipkart.foxtrot.core.common.noncacheable; import com.flipkart.foxtrot.common.ActionResponse; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.core.common.Action; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.flipkart.foxtrot.core.exception.MalformedQueryException; @@ -24,6 +25,8 @@ import com.google.common.annotations.VisibleForTesting; import org.elasticsearch.action.ActionRequest; +import java.util.List; + /** * Created by rishabh.goyal on 02/05/14. */ @@ -57,7 +60,7 @@ public void validateImpl(NonCacheableActionRequest parameter) throws MalformedQu } @Override - public ActionRequest getRequestBuilder(NonCacheableActionRequest parameter) throws FoxtrotException { + public ActionRequest getRequestBuilder(NonCacheableActionRequest parameter, List extraFilters) throws FoxtrotException { return null; } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java index b06d7e6b9..9116d3720 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java @@ -22,6 +22,7 @@ import com.flipkart.foxtrot.common.count.CountRequest; import com.flipkart.foxtrot.common.count.CountResponse; import com.flipkart.foxtrot.common.query.*; +import com.flipkart.foxtrot.common.query.general.EqualsFilter; import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.google.common.collect.Maps; @@ -30,10 +31,7 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -86,6 +84,43 @@ public void testMultiQuery() throws FoxtrotException, JsonProcessingException { assertEquals(9, countResponse.getCount()); } + @Test + public void testMultiQueryCommonFilters() throws FoxtrotException, JsonProcessingException { + + HashMap requests = Maps.newHashMap(); + Query query = new Query(); + query.setTable(TestUtils.TEST_TABLE_NAME); + ResultSort resultSort = new ResultSort(); + resultSort.setOrder(ResultSort.Order.asc); + resultSort.setField("_timestamp"); + query.setSort(resultSort); + requests.put("1", query); + + CountRequest countRequest = new CountRequest(); + countRequest.setTable(TestUtils.TEST_TABLE_NAME); + countRequest.setField("os"); + countRequest.setDistinct(false); + requests.put("2", countRequest); + + MultiQueryRequest multiQueryRequest = new MultiQueryRequest(requests); + multiQueryRequest.setFilters(Collections.singletonList(new EqualsFilter("device", "nexus"))); + + ActionResponse actionResponse = getQueryExecutor().execute(multiQueryRequest); + MultiQueryResponse multiQueryResponse = null; + if(actionResponse instanceof MultiQueryResponse) { + multiQueryResponse = (MultiQueryResponse)actionResponse; + } + assertNotNull(multiQueryResponse); + + QueryResponse queryResponse = (QueryResponse)multiQueryResponse.getResponses() + .get("1"); + CountResponse countResponse = (CountResponse)multiQueryResponse.getResponses() + .get("2"); + + assertEquals(6, queryResponse.getDocuments().size()); + assertEquals(6, countResponse.getCount()); + } + @Test public void testQueryNoFilterAscending() throws FoxtrotException, JsonProcessingException { HashMap requests = Maps.newHashMap(); diff --git a/foxtrot-server/src/main/resources/console/js/hostdetails.js b/foxtrot-server/src/main/resources/console/js/hostdetails.js index 4496f1c3f..f337192c1 100644 --- a/foxtrot-server/src/main/resources/console/js/hostdetails.js +++ b/foxtrot-server/src/main/resources/console/js/hostdetails.js @@ -14,7 +14,7 @@ * limitations under the License. */ - function HostDetails(hostname, port) { +function HostDetails(hostname, port) { this.hostname = hostname; this.port = port; } From 994162f2e8e1c907ac081785505b41fc100f5b89 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Fri, 8 May 2020 11:52:35 +0530 Subject: [PATCH 20/37] Added filters cache key and enforcing no overlap --- .../querystore/actions/MultiQueryAction.java | 52 ++++++++++++++++--- .../core/querystore/actions/Utils.java | 7 +++ .../actions/MultiQueryActionTest.java | 8 +-- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java index def05625f..e610895c4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java @@ -15,26 +15,27 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; import org.glassfish.hk2.api.MultiException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /*** Created by nitish.goyal on 22/08/18 ***/ @AnalyticsProvider(opcode = "multi_query", request = MultiQueryRequest.class, response = MultiQueryResponse.class, cacheable = true) +@Slf4j public class MultiQueryAction extends Action { - private static final Logger LOGGER = LoggerFactory.getLogger(MultiQueryAction.class); private AnalyticsLoader analyticsLoader; private Map requestActionMap = Maps.newHashMap(); @@ -60,8 +61,17 @@ public String getMetricKey() { @Override public String getRequestCacheKey() { - createActions(getParameter()); - return processForSubQueries(getParameter(), (action, request) -> action.getRequestCacheKey()); + final MultiQueryRequest parameter = getParameter(); + createActions(parameter); + long filterHashKey = 0L; + if (null != parameter.getFilters()) { + for (Filter filter : parameter.getFilters()) { + filterHashKey += 31 * filter.hashCode(); + } + } + return String.format("multquery-%d-%s", + filterHashKey, + processForSubQueries(parameter, (action, request) -> action.getRequestCacheKey())); } @Override @@ -84,12 +94,24 @@ public void validateImpl(MultiQueryRequest parameter) { @Override public ActionResponse execute(MultiQueryRequest parameter) { + if(Utils.hasTemporalFilters(parameter.getFilters())) { + val offendingRequests = parameter.getRequests().entrySet().stream() + .filter(entry -> Utils.hasTemporalFilters(entry.getValue().getFilters())) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(offendingRequests)) { + throw FoxtrotExceptions.createMalformedQueryException( + parameter, + Collections.singletonList( + "Temporal filters passed in multi query as well as children: " + offendingRequests)); + } + } MultiSearchRequest multiSearchRequestBuilder = getRequestBuilder(parameter, Collections.emptyList()); try { - LOGGER.info("Search: {}", multiSearchRequestBuilder); + log.info("Search: {}", multiSearchRequestBuilder); MultiSearchResponse multiSearchResponse = getConnection() .getClient() - .multiSearch(multiSearchRequestBuilder); + .multiSearch(multiSearchRequestBuilder, RequestOptions.DEFAULT); return getResponse(multiSearchResponse, parameter); } catch (IOException e) { throw FoxtrotExceptions.createQueryExecutionException(parameter, e); @@ -142,7 +164,7 @@ public ActionResponse getResponse(org.elasticsearch.action.ActionResponse multiS try { action = analyticsLoader.getAction(request); } catch (Exception e) { - LOGGER.error("Error occurred while executing multiQuery request : {}", e); + log.error("Error occurred while executing multiQuery request : {}", e); } if(null == action) { throw FoxtrotExceptions.queryCreationException(request, null); @@ -155,6 +177,19 @@ public ActionResponse getResponse(org.elasticsearch.action.ActionResponse multiS } private void createActions(final MultiQueryRequest multiQueryRequest) { + if(Utils.hasTemporalFilters(multiQueryRequest.getFilters())) { + val offendingRequests = multiQueryRequest.getRequests().entrySet().stream() + .filter(entry -> Utils.hasTemporalFilters(entry.getValue().getFilters())) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(offendingRequests)) { + throw FoxtrotExceptions.createMalformedQueryException( + multiQueryRequest, + Collections.singletonList( + "Temporal filters passed in multi query as well as children: " + offendingRequests)); + } + } + for(Map.Entry entry : multiQueryRequest.getRequests().entrySet()) { ActionRequest request = entry.getValue(); Action action; @@ -169,6 +204,7 @@ private void createActions(final MultiQueryRequest multiQueryRequest) { throw FoxtrotExceptions.createMalformedQueryException(multiQueryRequest, Collections.singletonList( "No action found for the sub request : " + request.toString())); } + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java index 15bcf98ae..78f745dc9 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java @@ -5,6 +5,7 @@ import com.flipkart.foxtrot.common.FieldType; import com.flipkart.foxtrot.common.Period; import com.flipkart.foxtrot.common.TableFieldMapping; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.common.query.ResultSort; import com.flipkart.foxtrot.common.stats.Stat; import com.flipkart.foxtrot.common.util.CollectionUtils; @@ -353,4 +354,10 @@ public static boolean isNumericField(TableMetadataManager tableMetadataManager, return null != fieldMetadata && NUMERIC_FIELD_TYPES.contains(fieldMetadata.getType()); } + public static boolean hasTemporalFilters(List filters) { + if(null == filters) { + return false; + } + return filters.stream().anyMatch(Filter::isFilterTemporal); + } } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java index 9116d3720..988e3ebb2 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java @@ -48,6 +48,7 @@ public static void setUp() throws Exception { getElasticsearchConnection().getClient() .indices() .refresh(new RefreshRequest("*"), RequestOptions.DEFAULT); + Thread.sleep(2000); } @Test @@ -81,6 +82,7 @@ public void testMultiQuery() throws FoxtrotException, JsonProcessingException { CountResponse countResponse = (CountResponse)multiQueryResponse.getResponses() .get("2"); + System.out.println(getMapper().writerWithDefaultPrettyPrinter().writeValueAsString(queryResponse)); assertEquals(9, countResponse.getCount()); } @@ -103,7 +105,7 @@ public void testMultiQueryCommonFilters() throws FoxtrotException, JsonProcessin requests.put("2", countRequest); MultiQueryRequest multiQueryRequest = new MultiQueryRequest(requests); - multiQueryRequest.setFilters(Collections.singletonList(new EqualsFilter("device", "nexus"))); + multiQueryRequest.setFilters(Collections.singletonList(new EqualsFilter("os", "ios"))); ActionResponse actionResponse = getQueryExecutor().execute(multiQueryRequest); MultiQueryResponse multiQueryResponse = null; @@ -117,8 +119,8 @@ public void testMultiQueryCommonFilters() throws FoxtrotException, JsonProcessin CountResponse countResponse = (CountResponse)multiQueryResponse.getResponses() .get("2"); - assertEquals(6, queryResponse.getDocuments().size()); - assertEquals(6, countResponse.getCount()); + assertEquals(2, queryResponse.getDocuments().size()); + assertEquals(2, countResponse.getCount()); } @Test From 04f69b7784afe3c60af05312e35132c5b4927fcd Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Tue, 12 May 2020 12:23:07 +0530 Subject: [PATCH 21/37] Null check --- .../foxtrot/core/querystore/actions/MultiQueryAction.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java index e610895c4..9fe768e44 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java @@ -211,6 +211,10 @@ private void createActions(final MultiQueryRequest multiQueryRequest) { private String processForSubQueries(MultiQueryRequest multiQueryRequest, ActionInterface actionInterface) { List results = Lists.newArrayList(); for(Map.Entry entry : multiQueryRequest.getRequests().entrySet()) { + if(null == entry.getValue()) { + log.warn("Empty response for query: {}", entry.getKey()); + continue; + } String result = actionInterface.invoke(requestActionMap.get(entry.getValue()), entry.getValue()); if(!Strings.isNullOrEmpty(result)) { results.add(result); From 545c744ed011550ec101464822fe03621ce65073 Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Mon, 1 Jun 2020 19:05:19 +0530 Subject: [PATCH 22/37] Build fix --- .../core/reroute/ClusterRerouteManager.java | 4 ++-- .../server/resources/ClusterHealthResource.java | 16 ++++------------ .../server/resources/ESClusterResource.java | 4 ++-- .../resources/ClusterHealthResourceTest.java | 7 ++----- 4 files changed, 10 insertions(+), 21 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteManager.java index 188f978fb..60f315953 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteManager.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteManager.java @@ -39,7 +39,7 @@ public ClusterRerouteManager(ElasticsearchConnection connection, ClusterRerouteC this.clusterRerouteConfig = clusterRerouteConfig; } - public void reallocate() { + /*public void reallocate() { Map nodeIdVsNodeInfoMap = new HashMap<>(); BiMap nodeNameVsNodeId = HashBiMap.create(); this.createNodeInfoMap(nodeIdVsNodeInfoMap); @@ -167,6 +167,6 @@ private Deque getVacantNodeId(int avgShardsPerNode, } } return vacantNodeIds; - } + }*/ } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java index e1f6faedf..8092e437a 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java @@ -49,24 +49,16 @@ public class ClusterHealthResource { private final QueryStore queryStore; - private final TableManager tableManager; - private final TableMetadataManager tableMetadataManager; @Inject - public ClusterHealthResource( - QueryStore queryStore, - TableManager tableManager, - TableMetadataManager tableMetadataManager) { + public ClusterHealthResource(QueryStore queryStore) { this.queryStore = queryStore; - this.tableManager = tableManager; - this.tableMetadataManager = tableMetadataManager; } - @GET @Timed @ApiOperation("getHealth") - public ClusterHealthResponse getHealth() throws ExecutionException, InterruptedException { + public ClusterHealthResponse getHealth() { return queryStore.getClusterHealth(); } @@ -74,7 +66,7 @@ public ClusterHealthResponse getHealth() throws ExecutionException, InterruptedE @Timed @Path("/nodestats") @ApiOperation("getNodeStat") - public JsonNode getNodeStat() throws ExecutionException, InterruptedException { + public JsonNode getNodeStat() { return queryStore.getNodeStats(); } @@ -83,7 +75,7 @@ public JsonNode getNodeStat() throws ExecutionException, InterruptedException { @Path("indicesstats") @ApiOperation("getIndicesStat") @SneakyThrows - public JsonNode getIndicesStat() throws ExecutionException, InterruptedException { + public JsonNode getIndicesStat() { return queryStore.getIndicesStats(); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ESClusterResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ESClusterResource.java index e87d58c27..935bdbb73 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ESClusterResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ESClusterResource.java @@ -31,10 +31,10 @@ public ESClusterResource(ClusterRerouteManager clusterRerouteManager) { this.clusterRerouteManager = clusterRerouteManager; } - @GET + /*@GET @Path("/reallocate") @ApiOperation("reallocate shards") public void reallocate() { clusterRerouteManager.reallocate(); - } + }*/ } diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/ClusterHealthResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/ClusterHealthResourceTest.java index a27ca4be4..50f2a87ee 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/ClusterHealthResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/ClusterHealthResourceTest.java @@ -30,19 +30,16 @@ import org.junit.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.UUID; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - /** * Created by swapnil on 25/01/16. */ public class ClusterHealthResourceTest extends FoxtrotResourceTest { - private final FoxtrotTableManager tableManager; + private FoxtrotTableManager tableManager; + @Rule public ResourceTestRule resources; From 1149c379ec92a3d66190dd0ac01c3585c760c83d Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Tue, 16 Jun 2020 16:07:13 +0530 Subject: [PATCH 23/37] Updated docker file --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d4d158a86..c0b52fe1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,5 +16,5 @@ ADD config/docker.yml docker.yml ADD foxtrot-server/target/foxtrot*.jar server.jar ADD scripts/local_es_setup.sh local_es_setup.sh -CMD sh -c "sleep 15 ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" +CMD sh -c "sleep 15 ; java -jar server.jar initialize docker.yml || true ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" From 15315a4c5c97d109f0e415a184614ae6c10f6ed2 Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Wed, 22 Jul 2020 10:33:41 +0530 Subject: [PATCH 24/37] Version update --- build.properties | 2 +- foxtrot-common/pom.xml | 2 +- foxtrot-core/pom.xml | 2 +- foxtrot-server/pom.xml | 2 +- foxtrot-sql/pom.xml | 2 +- foxtrot-translator/pom.xml | 2 +- pom.xml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build.properties b/build.properties index c1fdb92cc..d440758b0 100644 --- a/build.properties +++ b/build.properties @@ -1 +1 @@ -BUILD_PROJECT_VERSION=6.3.1-8 \ No newline at end of file +BUILD_PROJECT_VERSION=6.3.1-9 \ No newline at end of file diff --git a/foxtrot-common/pom.xml b/foxtrot-common/pom.xml index 7221bbe3f..180266c54 100644 --- a/foxtrot-common/pom.xml +++ b/foxtrot-common/pom.xml @@ -5,7 +5,7 @@ com.flipkart.foxtrot foxtrot - 6.3.1-8 + 6.3.1-9 4.0.0 diff --git a/foxtrot-core/pom.xml b/foxtrot-core/pom.xml index b96b1c913..6e6ec5a5d 100644 --- a/foxtrot-core/pom.xml +++ b/foxtrot-core/pom.xml @@ -5,7 +5,7 @@ com.flipkart.foxtrot foxtrot - 6.3.1-8 + 6.3.1-9 4.0.0 diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 0a477ecc6..2ed56f1a1 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -5,7 +5,7 @@ foxtrot com.flipkart.foxtrot - 6.3.1-8 + 6.3.1-9 4.0.0 diff --git a/foxtrot-sql/pom.xml b/foxtrot-sql/pom.xml index 3611218b5..5aac150cf 100644 --- a/foxtrot-sql/pom.xml +++ b/foxtrot-sql/pom.xml @@ -5,7 +5,7 @@ foxtrot com.flipkart.foxtrot - 6.3.1-8 + 6.3.1-9 4.0.0 diff --git a/foxtrot-translator/pom.xml b/foxtrot-translator/pom.xml index 664543e3a..ab4d8b659 100644 --- a/foxtrot-translator/pom.xml +++ b/foxtrot-translator/pom.xml @@ -5,7 +5,7 @@ foxtrot com.flipkart.foxtrot - 6.3.1-8 + 6.3.1-9 4.0.0 diff --git a/pom.xml b/pom.xml index 11216e7a3..9d01fdce2 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ com.flipkart.foxtrot foxtrot pom - 6.3.1-8 + 6.3.1-9 scm:git:https://github.com/flipkart-incubator/foxtrot.git From ca87638f9d18bbdcfa456c277c85f86f923fde68 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 22 Jul 2020 11:27:07 +0530 Subject: [PATCH 25/37] Updates Hzelcast to use new aws service discovery --- Dockerfile | 4 +- config/docker.yml | 6 +- foxtrot-core/pom.xml | 4 +- .../core/cache/impl/DistributedCache.java | 2 +- .../cache/impl/DistributedCacheFactory.java | 18 ++---- .../core/datastore/impl/hbase/HBaseUtil.java | 22 +------ .../impl/AwsECSDiscoveryConfig.java | 43 ++++++++++++++ .../querystore/impl/ClusterDiscoveryType.java | 1 + .../querystore/impl/HazelcastConnection.java | 59 ++++++++++++++----- .../impl/DistributedTableMetadataManager.java | 7 ++- .../core/table/impl/TableMapStore.java | 4 +- .../DistributedTableMetadataManagerTest.java | 2 +- foxtrot-server/pom.xml | 2 +- .../foxtrot/server/FoxtrotServer.java | 6 +- .../foxtrot/server/InitializerCommand.java | 1 + .../DistributedSessionDataStore.java | 2 +- .../server/cluster/ClusterManager.java | 6 +- .../server/resources/ClusterInfoResource.java | 2 +- pom.xml | 1 + 19 files changed, 122 insertions(+), 70 deletions(-) create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java diff --git a/Dockerfile b/Dockerfile index 5d6a19044..9ad79a662 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,9 +12,9 @@ EXPOSE 5701 VOLUME /var/log/foxtrot-server -ADD config/docker.yml docker.yml +ADD config/docker.yml config/docker.yml ADD foxtrot-server/target/foxtrot-server*.jar server.jar ADD scripts/local_es_setup.sh local_es_setup.sh -CMD sh -c "sleep 15 ; java -jar server.jar initialize docker.yml || true ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" +CMD sh -c "java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server ${CONFIG-PATH-config/docker.yml}" diff --git a/config/docker.yml b/config/docker.yml index 6e6b76ff2..188a0d03d 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -26,9 +26,9 @@ cluster: name: foxtrot discovery: type: ${DISCOVERY_TYPE} - disableMulticast: true - members: ["localhost:5701"] - +# disableMulticast: true +# members: ["${HOSTNAME}:5701"] + network: ${NETWORK} cacheConfig: maxIdleSeconds: 15 timeToLiveSeconds: 15 diff --git a/foxtrot-core/pom.xml b/foxtrot-core/pom.xml index b96b1c913..36caca8c2 100644 --- a/foxtrot-core/pom.xml +++ b/foxtrot-core/pom.xml @@ -170,7 +170,7 @@ com.hazelcast hazelcast - 3.11.7 + ${hazelcast.version} @@ -182,7 +182,7 @@ com.hazelcast hazelcast - 3.11.7 + ${hazelcast.version} test-jar test diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCache.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCache.java index b07af47c5..8733cb81e 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCache.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCache.java @@ -21,7 +21,7 @@ import com.flipkart.foxtrot.core.cache.Cache; import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; import com.flipkart.foxtrot.core.util.Constants; -import com.hazelcast.core.IMap; +import com.hazelcast.map.IMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCacheFactory.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCacheFactory.java index a6cedf8c9..da6ee8ba5 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCacheFactory.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCacheFactory.java @@ -20,10 +20,7 @@ import com.flipkart.foxtrot.core.cache.CacheFactory; import com.flipkart.foxtrot.core.querystore.impl.CacheConfig; import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; -import com.hazelcast.config.EvictionPolicy; -import com.hazelcast.config.InMemoryFormat; -import com.hazelcast.config.MapConfig; -import com.hazelcast.config.MaxSizeConfig; +import com.hazelcast.config.*; import javax.inject.Inject; import javax.inject.Singleton; @@ -60,7 +57,10 @@ private MapConfig getDefaultMapConfig(CacheConfig cacheConfig) { MapConfig mapConfig = new MapConfig(CACHE_NAME_PREFIX + "*"); mapConfig.setInMemoryFormat(InMemoryFormat.BINARY); mapConfig.setBackupCount(0); - mapConfig.setEvictionPolicy(EvictionPolicy.LRU); + final EvictionConfig evictionConfig = mapConfig.getEvictionConfig(); + evictionConfig.setEvictionPolicy(EvictionPolicy.LRU); + evictionConfig.setSize(cacheConfig.getSize() == 0 ? DEFAULT_SIZE : cacheConfig.getSize()); + evictionConfig.setMaxSizePolicy(MaxSizePolicy.USED_HEAP_SIZE); if(cacheConfig.getMaxIdleSeconds() == 0) { mapConfig.setMaxIdleSeconds(DEFAULT_MAX_IDLE_SECONDS); @@ -74,14 +74,6 @@ private MapConfig getDefaultMapConfig(CacheConfig cacheConfig) { mapConfig.setTimeToLiveSeconds(cacheConfig.getTimeToLiveSeconds()); } - MaxSizeConfig maxSizeConfig = new MaxSizeConfig(); - maxSizeConfig.setMaxSizePolicy(MaxSizeConfig.MaxSizePolicy.USED_HEAP_PERCENTAGE); - if(cacheConfig.getSize() == 0) { - maxSizeConfig.setSize(DEFAULT_SIZE); - } else { - maxSizeConfig.setSize(cacheConfig.getSize()); - } - mapConfig.setMaxSizeConfig(maxSizeConfig); return mapConfig; } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java index 4c72a08ea..95bd75e14 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java @@ -47,30 +47,12 @@ public static void createTable(final HbaseConfig hbaseConfig, final String table hTableDescriptor.addFamily(columnDescriptor); Configuration configuration = HBaseUtil.create(hbaseConfig); Connection connection = ConnectionFactory.createConnection(configuration); - Admin hBaseAdmin = null; - try { - hBaseAdmin = connection.getAdmin(); - hBaseAdmin.createTable(hTableDescriptor); + try(Admin admin = connection.getAdmin()) { + admin.createTable(hTableDescriptor); } catch (Exception e) { logger.error("Could not create table: " + tableName, e); } - finally { - try { - if (hBaseAdmin != null) { - hBaseAdmin.close(); - } - } - catch (Exception e) { - logger.error("Error closing hbase admin", e); - } - try { - connection.close(); - } - catch (Exception e) { - logger.error("Error closing hbase connection", e); - } - } } public static Configuration create(final HbaseConfig hbaseConfig) throws IOException { diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java new file mode 100644 index 000000000..230b8f37f --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java @@ -0,0 +1,43 @@ +/** + * Copyright 2016 Flipkart Internet Pvt. Ltd. + *

      + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.foxtrot.core.querystore.impl; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.hibernate.validator.constraints.NotEmpty; + +/** + * @author phaneesh + */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class AwsECSDiscoveryConfig extends ClusterDiscoveryConfig { + + @NotEmpty + @JsonProperty + private String network; + + @JsonProperty + private String serviceName; + + public AwsECSDiscoveryConfig() { + super(ClusterDiscoveryType.FOXTROT_AWS_ECS); + } + +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ClusterDiscoveryType.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ClusterDiscoveryType.java index 60817757a..aacdcb8d4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ClusterDiscoveryType.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ClusterDiscoveryType.java @@ -19,5 +19,6 @@ public enum ClusterDiscoveryType { FOXTROT_SIMPLE, FOXTROT_MARATHON, FOXTROT_AWS, + FOXTROT_AWS_ECS, FOXTROT_KUBERNETES } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java index 392ecc71a..57cf2e2e8 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java @@ -15,10 +15,12 @@ */ package com.flipkart.foxtrot.core.querystore.impl; +import com.google.common.base.Strings; +import com.hazelcast.client.properties.ClientProperty; import com.hazelcast.config.*; import com.hazelcast.core.Hazelcast; import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.spi.properties.GroupProperty; +import com.hazelcast.spi.properties.ClusterProperty; import com.marathon.hazelcast.servicediscovery.MarathonDiscoveryStrategyFactory; import io.dropwizard.lifecycle.Managed; import org.slf4j.Logger; @@ -48,8 +50,7 @@ public class HazelcastConnection implements Managed { @Inject public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostException { Config hzConfig = new Config(); - hzConfig.getGroupConfig() - .setName(clusterConfig.getName()); + hzConfig.setClusterName(clusterConfig.getName()); switch (clusterConfig.getDiscovery() .getType()) { case FOXTROT_SIMPLE: @@ -81,14 +82,12 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept String appId = marathonClusterDiscoveryConfig.getApp() .replace("/", "") .trim(); - hzConfig.getGroupConfig() - .setName("foxtrot"); - hzConfig.getGroupConfig() - .setPassword("foxtrot"); - hzConfig.setProperty(GroupProperty.DISCOVERY_SPI_ENABLED.getName(), "true"); - hzConfig.setProperty(GroupProperty.DISCOVERY_SPI_PUBLIC_IP_ENABLED.getName(), "true"); - hzConfig.setProperty(GroupProperty.SOCKET_CLIENT_BIND_ANY.getName(), "true"); - hzConfig.setProperty(GroupProperty.SOCKET_BIND_ANY.getName(), "true"); + hzConfig.setClusterName("foxtrot"); + hzConfig.setProperty("hazelcast.application.validation.token", "foxtrot"); + hzConfig.setProperty(ClientProperty.DISCOVERY_SPI_ENABLED.getName(), "true"); + hzConfig.setProperty(ClientProperty.DISCOVERY_SPI_PUBLIC_IP_ENABLED.getName(), "true"); + hzConfig.setProperty(ClusterProperty.SOCKET_CLIENT_BIND_ANY.getName(), "true"); + hzConfig.setProperty(ClusterProperty.SOCKET_BIND_ANY.getName(), "true"); NetworkConfig networkConfig = hzConfig.getNetworkConfig(); networkConfig.setPublicAddress(System.getenv("HOST") + ":" + System.getenv("PORT_5701")); @@ -107,7 +106,7 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept discoveryStrategyConfig.addProperty("port-index", marathonClusterDiscoveryConfig.getPortIndex()); discoveryConfig.addDiscoveryStrategyConfig(discoveryStrategyConfig); break; - case FOXTROT_AWS: + case FOXTROT_AWS: { AwsClusterDiscoveryConfig awsClusterDiscoveryConfig = (AwsClusterDiscoveryConfig) clusterConfig.getDiscovery(); NetworkConfig hazelcastConfigNetworkConfig = hzConfig.getNetworkConfig(); JoinConfig hazelcastConfigNetworkConfigJoin = hazelcastConfigNetworkConfig.getJoin(); @@ -116,7 +115,7 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept hazelcastConfigNetworkConfigJoin.getMulticastConfig() .setEnabled(false); AwsConfig awsConfig = new AwsConfig(); - awsConfig.setAccessKey(awsClusterDiscoveryConfig.getAccessKey()); +/* awsConfig.setAccessKey(awsClusterDiscoveryConfig.getAccessKey()); awsConfig.setConnectionTimeoutSeconds(awsClusterDiscoveryConfig.getConnectionTimeoutSeconds()); awsConfig.setHostHeader(awsClusterDiscoveryConfig.getHostHeader()); awsConfig.setIamRole(awsClusterDiscoveryConfig.getIamRole()); @@ -124,11 +123,43 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept awsConfig.setSecurityGroupName(awsClusterDiscoveryConfig.getSecurityGroupName()); awsConfig.setSecretKey(awsClusterDiscoveryConfig.getSecretKey()); awsConfig.setTagKey(awsClusterDiscoveryConfig.getTagKey()); - awsConfig.setTagValue(awsClusterDiscoveryConfig.getTagValue()); + awsConfig.setTagValue(awsClusterDiscoveryConfig.getTagValue());*/ hazelcastConfigNetworkConfigJoin.setAwsConfig(awsConfig); hazelcastConfigNetworkConfigJoin.getAwsConfig() .setEnabled(true); break; + } + case FOXTROT_AWS_ECS: + AwsECSDiscoveryConfig awsECSDiscoveryConfig = (AwsECSDiscoveryConfig) clusterConfig.getDiscovery(); + NetworkConfig hazelcastConfigNetworkConfig = hzConfig.getNetworkConfig(); +// JoinConfig hazelcastConfigNetworkConfigJoin = hazelcastConfigNetworkConfig.getJoin(); + hazelcastConfigNetworkConfig.getJoin() + .getMulticastConfig() + .setEnabled(false); + + hazelcastConfigNetworkConfig.getInterfaces() + .setEnabled(true) + .addInterface(awsECSDiscoveryConfig.getNetwork()); + + AwsConfig awsConfig = new AwsConfig(); + awsConfig.setEnabled(true); + if(!Strings.isNullOrEmpty(awsECSDiscoveryConfig.getServiceName())) { + awsConfig.setProperty("service-name", awsECSDiscoveryConfig.getServiceName()); + } + +/* awsConfig.setAccessKey(awsECSDiscoveryConfig.getAccessKey()); + awsConfig.setConnectionTimeoutSeconds(awsECSDiscoveryConfig.getConnectionTimeoutSeconds()); + awsConfig.setHostHeader(awsECSDiscoveryConfig.getHostHeader()); + awsConfig.setIamRole(awsECSDiscoveryConfig.getIamRole()); + awsConfig.setRegion(awsECSDiscoveryConfig.getRegion()); + awsConfig.setSecurityGroupName(awsECSDiscoveryConfig.getSecurityGroupName()); + awsConfig.setSecretKey(awsECSDiscoveryConfig.getSecretKey()); + awsConfig.setTagKey(awsECSDiscoveryConfig.getTagKey()); + awsConfig.setTagValue(awsECSDiscoveryConfig.getTagValue()); + hazelcastConfigNetworkConfigJoin.setAwsConfig(awsConfig); + hazelcastConfigNetworkConfigJoin.getAwsConfig() + .setEnabled(true);*/ + break; case FOXTROT_KUBERNETES: logger.info("Using Kubernetes"); JoinConfig kbConfig = hzConfig.getNetworkConfig().getJoin(); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java index 43b0de418..0bf22bca7 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java @@ -42,12 +42,15 @@ import com.hazelcast.config.MapConfig; import com.hazelcast.config.MapStoreConfig; import com.hazelcast.config.NearCacheConfig; -import com.hazelcast.core.IMap; +import com.hazelcast.map.IMap; import lombok.SneakyThrows; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.*; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.TimeValue; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java index 4f9134095..f9ab4ee40 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java @@ -23,8 +23,8 @@ import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.hazelcast.core.MapStore; -import com.hazelcast.core.MapStoreFactory; +import com.hazelcast.map.MapStore; +import com.hazelcast.map.MapStoreFactory; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.*; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java index 42a39a85e..fdef73717 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java @@ -33,7 +33,7 @@ import com.google.common.collect.Lists; import com.hazelcast.config.Config; import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.core.IMap; +import com.hazelcast.map.IMap; import com.hazelcast.test.TestHazelcastInstanceFactory; import org.joda.time.DateTime; import org.junit.*; diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 0a477ecc6..eb33babda 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -99,7 +99,7 @@ com.hazelcast hazelcast - 3.11.7 + ${hazelcast.version} test-jar test diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java index 9f623264f..c530fbd19 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java @@ -19,10 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.impl.KubernetesClusterDiscoveryConfig; -import com.flipkart.foxtrot.core.querystore.impl.MarathonClusterDiscoveryConfig; -import com.flipkart.foxtrot.core.querystore.impl.SimpleClusterDiscoveryConfig; +import com.flipkart.foxtrot.core.querystore.impl.*; import com.flipkart.foxtrot.core.util.MetricUtil; import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; import com.flipkart.foxtrot.server.di.FoxtrotModule; @@ -125,6 +122,7 @@ private void configureObjectMapper(ObjectMapper objectMapper) { objectMapper.registerSubtypes(new NamedType(SimpleClusterDiscoveryConfig.class, "foxtrot_simple")); objectMapper.registerSubtypes(new NamedType(MarathonClusterDiscoveryConfig.class, "foxtrot_marathon")); objectMapper.registerSubtypes(new NamedType(KubernetesClusterDiscoveryConfig.class, "foxtrot_kubernetes")); + objectMapper.registerSubtypes(new NamedType(AwsECSDiscoveryConfig.class, "foxtrot_aws_ecs")); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java index 2319105c7..ecde0ea14 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java @@ -79,6 +79,7 @@ protected void run(Bootstrap bootstrap, Namespace na logger.info("Creating hbase table"); HBaseUtil.createTable(configuration.getHbase(), configuration.getHbase() .getTableName()); + logger.info("Initialization complete..."); } private void createMetaIndex(final ElasticsearchConnection connection, final String indexName, int replicaCount) { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java index 125555f91..57855a133 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java @@ -5,7 +5,7 @@ import com.hazelcast.config.InMemoryFormat; import com.hazelcast.config.MapConfig; import com.hazelcast.config.NearCacheConfig; -import com.hazelcast.core.IMap; +import com.hazelcast.map.IMap; import io.dropwizard.lifecycle.Managed; import ru.vyarus.dropwizard.guice.module.installer.order.Order; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java index a95a1d93f..0d66fc318 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java @@ -6,10 +6,10 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.hazelcast.cluster.Member; import com.hazelcast.config.EvictionPolicy; import com.hazelcast.config.MapConfig; -import com.hazelcast.core.IMap; -import com.hazelcast.core.Member; +import com.hazelcast.map.IMap; import io.dropwizard.lifecycle.Managed; import io.dropwizard.server.ServerFactory; import org.slf4j.Logger; @@ -47,7 +47,7 @@ public ClusterManager(HazelcastConnection connection, List healthCh mapConfig.setTimeToLiveSeconds(MAP_REFRESH_TIME + 2); //Reduce jitter mapConfig.setBackupCount(1); mapConfig.setAsyncBackupCount(2); - mapConfig.setEvictionPolicy(EvictionPolicy.NONE); + mapConfig.getEvictionConfig().setEvictionPolicy(EvictionPolicy.NONE); hazelcastConnection.getHazelcastConfig() .getMapConfigs() .put(MAP_NAME, mapConfig); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java index 16bcb66ec..f7024ed2c 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java @@ -3,7 +3,7 @@ import com.codahale.metrics.annotation.Timed; import com.flipkart.foxtrot.server.cluster.ClusterManager; import com.flipkart.foxtrot.server.cluster.ClusterMember; -import com.hazelcast.core.Member; +import com.hazelcast.cluster.Member; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; diff --git a/pom.xml b/pom.xml index 11216e7a3..baf02abad 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ ${project.basedir}/target/jacoco.exec java **com/flipkart/foxtrot/common/**/* + 4.0.2 From 2358e47b0a457182ed7d28c58ca760e171f1ff33 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 22 Jul 2020 23:27:11 +0530 Subject: [PATCH 26/37] Allowed external loading for docker file --- Dockerfile | 3 +- config/docker.yml | 6 +- docker-compose.yml | 1 + .../core/datastore/impl/hbase/HBaseUtil.java | 18 ++- .../impl/AwsClusterDiscoveryConfig.java | 106 +++++------------- .../impl/AwsECSDiscoveryConfig.java | 30 ++++- .../querystore/impl/HazelcastConnection.java | 92 ++++++++++----- .../foxtrot/server/InitializerCommand.java | 67 ++++++----- startup.sh | 34 ++++++ 9 files changed, 212 insertions(+), 145 deletions(-) create mode 100755 startup.sh diff --git a/Dockerfile b/Dockerfile index 9ad79a662..df3633a2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ VOLUME /var/log/foxtrot-server ADD config/docker.yml config/docker.yml ADD foxtrot-server/target/foxtrot-server*.jar server.jar ADD scripts/local_es_setup.sh local_es_setup.sh +ADD startup.sh startup.sh -CMD sh -c "java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server ${CONFIG-PATH-config/docker.yml}" +CMD ./startup.sh diff --git a/config/docker.yml b/config/docker.yml index 188a0d03d..440e2acac 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -26,9 +26,9 @@ cluster: name: foxtrot discovery: type: ${DISCOVERY_TYPE} -# disableMulticast: true -# members: ["${HOSTNAME}:5701"] - network: ${NETWORK} + disableMulticast: true + members: ["${HOSTNAME}:5701"] + cacheConfig: maxIdleSeconds: 15 timeToLiveSeconds: 15 diff --git a/docker-compose.yml b/docker-compose.yml index fa57ea60f..e9c307105 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,7 @@ services: volumes: - /var/log/foxtrot-server environment: + - INIT_SLEEP=15 - HOST=localhost - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9300 diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java index 95bd75e14..2b285867a 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java @@ -46,12 +46,18 @@ public static void createTable(final HbaseConfig hbaseConfig, final String table columnDescriptor.setCompressionType(Compression.Algorithm.GZ); hTableDescriptor.addFamily(columnDescriptor); Configuration configuration = HBaseUtil.create(hbaseConfig); - Connection connection = ConnectionFactory.createConnection(configuration); - try(Admin admin = connection.getAdmin()) { - admin.createTable(hTableDescriptor); - } - catch (Exception e) { - logger.error("Could not create table: " + tableName, e); + try(Connection connection = ConnectionFactory.createConnection(configuration)) { + try (Admin admin = connection.getAdmin()) { + if (admin.tableExists(TableName.valueOf(tableName))) { + logger.info("Table {} already exists. Nothing to do.", tableName); + return; + } + logger.info("Creating table: {}", tableName); + admin.createTable(hTableDescriptor); + } + catch (Exception e) { + logger.error("Could not create table: " + tableName, e); + } } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsClusterDiscoveryConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsClusterDiscoveryConfig.java index 8fa57489a..1d24ef1d6 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsClusterDiscoveryConfig.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsClusterDiscoveryConfig.java @@ -16,104 +16,50 @@ package com.flipkart.foxtrot.core.querystore.impl; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +import javax.validation.constraints.Min; /** - * @author phaneesh + * AWS EC2 based cluster configuration. + * See: https://github.com/hazelcast/hazelcast-aws#ecsfargate-configuration */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) public class AwsClusterDiscoveryConfig extends ClusterDiscoveryConfig { + @JsonProperty + private String serviceName; @JsonProperty private String accessKey; + @JsonProperty private String secretKey; + @JsonProperty - private String region = "us-east-1"; - @JsonProperty - private String securityGroupName; + private String iamRole; + @JsonProperty - private String tagKey; + private String region; + @JsonProperty - private String tagValue; + private String hostHeader; + @JsonProperty - private String hostHeader = "ec2.amazonaws.com"; + private String securityGroupName; + @JsonProperty - private String iamRole; + @Min(0) + private int opTimeoutSeconds; + @JsonProperty - private int connectionTimeoutSeconds = 5; + private boolean isExternalClient; public AwsClusterDiscoveryConfig() { super(ClusterDiscoveryType.FOXTROT_AWS); } - public String getAccessKey() { - return accessKey; - } - - public void setAccessKey(String accessKey) { - this.accessKey = accessKey; - } - - public String getSecretKey() { - return secretKey; - } - - public void setSecretKey(String secretKey) { - this.secretKey = secretKey; - } - - public String getRegion() { - return region; - } - - public void setRegion(String region) { - this.region = region; - } - - public String getSecurityGroupName() { - return securityGroupName; - } - - public void setSecurityGroupName(String securityGroupName) { - this.securityGroupName = securityGroupName; - } - - public String getTagKey() { - return tagKey; - } - - public void setTagKey(String tagKey) { - this.tagKey = tagKey; - } - - public String getTagValue() { - return tagValue; - } - - public void setTagValue(String tagValue) { - this.tagValue = tagValue; - } - - public String getHostHeader() { - return hostHeader; - } - - public void setHostHeader(String hostHeader) { - this.hostHeader = hostHeader; - } - - public String getIamRole() { - return iamRole; - } - - public void setIamRole(String iamRole) { - this.iamRole = iamRole; - } - - public int getConnectionTimeoutSeconds() { - return connectionTimeoutSeconds; - } - - public void setConnectionTimeoutSeconds(int connectionTimeoutSeconds) { - this.connectionTimeoutSeconds = connectionTimeoutSeconds; - } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java index 230b8f37f..a8515171b 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java @@ -21,8 +21,11 @@ import lombok.ToString; import org.hibernate.validator.constraints.NotEmpty; +import javax.validation.constraints.Min; + /** - * @author phaneesh + * AWS ECS based cluster configuration. + * See: https://github.com/hazelcast/hazelcast-aws#ecsfargate-configuration */ @Data @EqualsAndHashCode(callSuper = true) @@ -33,9 +36,34 @@ public class AwsECSDiscoveryConfig extends ClusterDiscoveryConfig { @JsonProperty private String network; + @JsonProperty + private String accessKey; + + @JsonProperty + private String secretKey; + + @JsonProperty + private String region; + + @JsonProperty + private String cluster; + + @JsonProperty + private String family; + @JsonProperty private String serviceName; + @JsonProperty + private String hostHeader; + + @JsonProperty + @Min(0) + private int opTimeoutSeconds; + + @JsonProperty + private boolean isExternalClient; + public AwsECSDiscoveryConfig() { super(ClusterDiscoveryType.FOXTROT_AWS_ECS); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java index 57cf2e2e8..444424b62 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java @@ -107,7 +107,7 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept discoveryConfig.addDiscoveryStrategyConfig(discoveryStrategyConfig); break; case FOXTROT_AWS: { - AwsClusterDiscoveryConfig awsClusterDiscoveryConfig = (AwsClusterDiscoveryConfig) clusterConfig.getDiscovery(); + AwsClusterDiscoveryConfig ec2Config = (AwsClusterDiscoveryConfig) clusterConfig.getDiscovery(); NetworkConfig hazelcastConfigNetworkConfig = hzConfig.getNetworkConfig(); JoinConfig hazelcastConfigNetworkConfigJoin = hazelcastConfigNetworkConfig.getJoin(); hazelcastConfigNetworkConfigJoin.getTcpIpConfig() @@ -115,22 +115,44 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept hazelcastConfigNetworkConfigJoin.getMulticastConfig() .setEnabled(false); AwsConfig awsConfig = new AwsConfig(); -/* awsConfig.setAccessKey(awsClusterDiscoveryConfig.getAccessKey()); - awsConfig.setConnectionTimeoutSeconds(awsClusterDiscoveryConfig.getConnectionTimeoutSeconds()); - awsConfig.setHostHeader(awsClusterDiscoveryConfig.getHostHeader()); - awsConfig.setIamRole(awsClusterDiscoveryConfig.getIamRole()); - awsConfig.setRegion(awsClusterDiscoveryConfig.getRegion()); - awsConfig.setSecurityGroupName(awsClusterDiscoveryConfig.getSecurityGroupName()); - awsConfig.setSecretKey(awsClusterDiscoveryConfig.getSecretKey()); - awsConfig.setTagKey(awsClusterDiscoveryConfig.getTagKey()); - awsConfig.setTagValue(awsClusterDiscoveryConfig.getTagValue());*/ + + if(!Strings.isNullOrEmpty(ec2Config.getServiceName())) { + awsConfig.setProperty("service-name", ec2Config.getServiceName()); + } + if(!Strings.isNullOrEmpty(ec2Config.getAccessKey())) { + awsConfig.setProperty("access-key", ec2Config.getAccessKey()); + } + if(!Strings.isNullOrEmpty(ec2Config.getSecretKey())) { + awsConfig.setProperty("secret-key", ec2Config.getSecretKey()); + } + if(!Strings.isNullOrEmpty(ec2Config.getIamRole())) { + awsConfig.setProperty("iam-role", ec2Config.getIamRole()); + } + if(!Strings.isNullOrEmpty(ec2Config.getRegion())) { + awsConfig.setProperty("region", ec2Config.getRegion()); + } + if(!Strings.isNullOrEmpty(ec2Config.getHostHeader())) { + awsConfig.setProperty("host-header", ec2Config.getHostHeader()); + } + if(!Strings.isNullOrEmpty(ec2Config.getSecurityGroupName())) { + awsConfig.setProperty("security-group-name", ec2Config.getSecurityGroupName()); + } + if(ec2Config.getOpTimeoutSeconds() > 0) { + awsConfig.setProperty("connection-timeout-seconds", + Integer.toString(ec2Config.getOpTimeoutSeconds())); + awsConfig.setProperty("read-timeout-seconds", + Integer.toString(ec2Config.getOpTimeoutSeconds())); + } + if(ec2Config.isExternalClient()) { + awsConfig.setProperty("use-public-ip", Boolean.TRUE.toString()); + } hazelcastConfigNetworkConfigJoin.setAwsConfig(awsConfig); hazelcastConfigNetworkConfigJoin.getAwsConfig() .setEnabled(true); break; } case FOXTROT_AWS_ECS: - AwsECSDiscoveryConfig awsECSDiscoveryConfig = (AwsECSDiscoveryConfig) clusterConfig.getDiscovery(); + AwsECSDiscoveryConfig ecsConfig = (AwsECSDiscoveryConfig) clusterConfig.getDiscovery(); NetworkConfig hazelcastConfigNetworkConfig = hzConfig.getNetworkConfig(); // JoinConfig hazelcastConfigNetworkConfigJoin = hazelcastConfigNetworkConfig.getJoin(); hazelcastConfigNetworkConfig.getJoin() @@ -139,26 +161,40 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept hazelcastConfigNetworkConfig.getInterfaces() .setEnabled(true) - .addInterface(awsECSDiscoveryConfig.getNetwork()); + .addInterface(ecsConfig.getNetwork()); AwsConfig awsConfig = new AwsConfig(); awsConfig.setEnabled(true); - if(!Strings.isNullOrEmpty(awsECSDiscoveryConfig.getServiceName())) { - awsConfig.setProperty("service-name", awsECSDiscoveryConfig.getServiceName()); - } - -/* awsConfig.setAccessKey(awsECSDiscoveryConfig.getAccessKey()); - awsConfig.setConnectionTimeoutSeconds(awsECSDiscoveryConfig.getConnectionTimeoutSeconds()); - awsConfig.setHostHeader(awsECSDiscoveryConfig.getHostHeader()); - awsConfig.setIamRole(awsECSDiscoveryConfig.getIamRole()); - awsConfig.setRegion(awsECSDiscoveryConfig.getRegion()); - awsConfig.setSecurityGroupName(awsECSDiscoveryConfig.getSecurityGroupName()); - awsConfig.setSecretKey(awsECSDiscoveryConfig.getSecretKey()); - awsConfig.setTagKey(awsECSDiscoveryConfig.getTagKey()); - awsConfig.setTagValue(awsECSDiscoveryConfig.getTagValue()); - hazelcastConfigNetworkConfigJoin.setAwsConfig(awsConfig); - hazelcastConfigNetworkConfigJoin.getAwsConfig() - .setEnabled(true);*/ + if(!Strings.isNullOrEmpty(ecsConfig.getAccessKey())) { + awsConfig.setProperty("access-key", ecsConfig.getAccessKey()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getSecretKey())) { + awsConfig.setProperty("secret-key", ecsConfig.getSecretKey()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getRegion())) { + awsConfig.setProperty("region", ecsConfig.getRegion()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getCluster())) { + awsConfig.setProperty("cluster", ecsConfig.getCluster()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getFamily())) { + awsConfig.setProperty("family", ecsConfig.getFamily()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getServiceName())) { + awsConfig.setProperty("service-name", ecsConfig.getServiceName()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getHostHeader())) { + awsConfig.setProperty("host-header", ecsConfig.getHostHeader()); + } + if(ecsConfig.getOpTimeoutSeconds() > 0) { + awsConfig.setProperty("connection-timeout-seconds", + Integer.toString(ecsConfig.getOpTimeoutSeconds())); + awsConfig.setProperty("read-timeout-seconds", + Integer.toString(ecsConfig.getOpTimeoutSeconds())); + } + if(ecsConfig.isExternalClient()) { + awsConfig.setProperty("use-public-ip", Boolean.TRUE.toString()); + } break; case FOXTROT_KUBERNETES: logger.info("Using Kubernetes"); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java index ecde0ea14..684aae73b 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java @@ -29,11 +29,12 @@ import net.sourceforge.argparse4j.inf.Namespace; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.common.settings.Settings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,49 +48,63 @@ public InitializerCommand() { } @Override - protected void run(Bootstrap bootstrap, Namespace namespace, FoxtrotServerConfiguration configuration) + protected void run(Bootstrap bootstrap, + Namespace namespace, + FoxtrotServerConfiguration configuration) throws Exception { ElasticsearchConfig esConfig = configuration.getElasticsearch(); ElasticsearchConnection connection = new ElasticsearchConnection(esConfig); connection.start(); - ClusterHealthResponse clusterHealth = connection.getClient() - .cluster() - .health(new ClusterHealthRequest(), RequestOptions.DEFAULT); - int numDataNodes = clusterHealth.getNumberOfDataNodes(); - int numReplicas = (numDataNodes < 2) ? 0 : 1; - - logger.info("# data nodes: {}, Setting replica count to: {}", numDataNodes, numReplicas); + try { + ClusterHealthResponse clusterHealth = connection.getClient() + .cluster() + .health(new ClusterHealthRequest(), RequestOptions.DEFAULT); + int numDataNodes = clusterHealth.getNumberOfDataNodes(); + int numReplicas = (numDataNodes < 2) + ? 0 + : 1; - createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX, numReplicas); - createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX_V2, numReplicas); - createMetaIndex(connection, TableMapStore.TABLE_META_INDEX, numReplicas); - createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX_HISTORY, numReplicas); - createMetaIndex(connection, FqlStoreServiceImpl.FQL_STORE_INDEX, numReplicas); - createMetaIndex(connection, "user-meta", numReplicas); - createMetaIndex(connection, "tokens", numReplicas); + logger.info("# data nodes: {}, Setting replica count to: {}", numDataNodes, numReplicas); - logger.info("Creating mapping"); - PutIndexTemplateRequest putIndexTemplateRequest = ElasticsearchUtils.getClusterTemplateMapping(); - AcknowledgedResponse response = connection.getClient() - .indices() - .putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); - logger.info("Created mapping: {}", response.isAcknowledged()); + createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX, numReplicas); + createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX_V2, numReplicas); + createMetaIndex(connection, TableMapStore.TABLE_META_INDEX, numReplicas); + createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX_HISTORY, numReplicas); + createMetaIndex(connection, FqlStoreServiceImpl.FQL_STORE_INDEX, numReplicas); + createMetaIndex(connection, "user-meta", numReplicas); + createMetaIndex(connection, "tokens", numReplicas); + logger.info("Creating mapping"); + PutIndexTemplateRequest putIndexTemplateRequest = ElasticsearchUtils.getClusterTemplateMapping(); + AcknowledgedResponse response = connection.getClient() + .indices() + .putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); + logger.info("Created mapping: {}", response.isAcknowledged()); + } + finally { + connection.stop(); + } logger.info("Creating hbase table"); - HBaseUtil.createTable(configuration.getHbase(), configuration.getHbase() - .getTableName()); + HBaseUtil.createTable(configuration.getHbase(), configuration.getHbase().getTableName()); logger.info("Initialization complete..."); } private void createMetaIndex(final ElasticsearchConnection connection, final String indexName, int replicaCount) { try { + if(connection.getClient() + .indices() + .exists(new GetIndexRequest(indexName), RequestOptions.DEFAULT)) { + logger.info("Index {} already exists. Nothing to do.", indexName); + return; + } + logger.info("'{}' creation started", indexName); Settings settings = Settings.builder() .put("number_of_shards", 1) .put("number_of_replicas", replicaCount) .build(); - CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(indexName) + CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName) .settings(settings); CreateIndexResponse response = connection.getClient() .indices() diff --git a/startup.sh b/startup.sh new file mode 100755 index 000000000..9155db9b9 --- /dev/null +++ b/startup.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +FOXTROT_CONFIG_FILE=/config/docker.yml + +if [ -z "${CONFIG_PATH}" ]; then + echo "No CONFIG_PATH defined. We shall be using default config from ${FOXTROT_CONFIG_FILE}" +else + if [ -f ${FOXTROT_CONFIG_FILE} ]; then + FOXTROT_CONFIG_FILE=${CONFIG_PATH} + else + echo "Defined CONFIG_PATH (${CONFIG_PATH}) doesn ot look like a proper file. Using default: ${FOXTROT_CONFIG_FILE}" + fi +fi + +if [ -z "${INIT_SLEEP}" ]; then + echo "No initial sleep specified.. Foxtrot will start up immediately" +else + echo -e "Init sleep of ${INIT_SLEEP} seconds specified. Waiting..." + sleep ${INIT_SLEEP} + echo "done" +fi + +INIT_COMMAND="java -jar server.jar initialize ${FOXTROT_CONFIG_FILE}" + +EXEC_CMD="java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server ${FOXTROT_CONFIG_FILE}" + +if [ -z "${SKIP_INIT}" ]; then + echo "Executing Init Command: ${INIT_COMMAND}" + $INIT_COMMAND +else + echo "Skipping Init as $SKIP_INIT was set" +fi +echo "Starting foxtrot with command line: ${EXEC_CMD}" +$EXEC_CMD From cdd0ba9341a0278ac049ae3303e46a85dc88b21a Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Fri, 29 Jan 2021 12:59:57 +0530 Subject: [PATCH 27/37] Remove ttl from table --- .../src/main/java/com/flipkart/foxtrot/common/Table.java | 1 - 1 file changed, 1 deletion(-) diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Table.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Table.java index 806190d05..2ced43036 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Table.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Table.java @@ -44,7 +44,6 @@ public class Table implements Serializable { private String name; @Min(1) - @Max(180) private int ttl; private boolean seggregatedBackend = false; From b0b2a5a8bc07e1b584cd2d743996b75e3216f639 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Tue, 6 Jul 2021 13:17:57 +0530 Subject: [PATCH 28/37] Working version of polymorphic auth --- Dockerfile | 2 +- GAUTH | 3 + config/docker.yml | 15 +- docker-compose.yml | 22 +- foxtrot-server/pom.xml | 11 + .../flipkart/foxtrot/server/AuthInfoType.java | 8 + .../foxtrot/server/auth/AuthInfo.java | 18 + .../foxtrot/server/auth/AuthInfoVisitor.java | 8 + .../server/auth/AuthenticatedInfo.java | 13 + .../foxtrot/server/auth/ESAuthStore.java | 3 +- .../foxtrot/server/auth/IdmanAuthStore.java | 86 ++ .../flipkart/foxtrot/server/auth/Token.java | 8 +- .../foxtrot/server/auth/TokenAuthInfo.java | 27 + .../server/auth/TokenAuthenticator.java | 47 +- .../auth/authprovider/AuthConfigVisitor.java | 3 + .../auth/authprovider/AuthProvider.java | 14 +- .../auth/authprovider/AuthProviderConfig.java | 2 + .../authprovider/AuthProviderFactory.java | 7 +- .../server/auth/authprovider/AuthType.java | 3 +- .../ConfiguredAuthProviderFactory.java | 36 + .../server/auth/authprovider/IdType.java | 9 + .../authprovider/impl/GoogleAuthProvider.java | 85 +- .../authprovider/impl/IdmanAuthProvider.java | 157 ++++ .../impl/IdmanAuthProviderConfig.java | 49 ++ .../auth/filter/UserAuthenticationFilter.java | 26 +- .../foxtrot/server/di/FoxtrotModule.java | 36 +- .../AuthenticationExceptionHandler.java | 2 +- .../resources/{GoogleAuth.java => OAuth.java} | 27 +- .../resources/console/echo/browse-events.htm | 1 + .../main/resources/console/echo/fql/index.htm | 1 + .../src/main/resources/console/echo/index.htm | 1 + out | 774 ++++++++++++++++++ scripts/cluster.json | 496 +++++++++++ 33 files changed, 1894 insertions(+), 106 deletions(-) create mode 100644 GAUTH create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/AuthInfoType.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfo.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfoVisitor.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthenticatedInfo.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthInfo.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/IdType.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProviderConfig.java rename foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/{GoogleAuth.java => OAuth.java} (84%) create mode 100644 out create mode 100644 scripts/cluster.json diff --git a/Dockerfile b/Dockerfile index 5d6a19044..b7ed0eaa7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,5 +16,5 @@ ADD config/docker.yml docker.yml ADD foxtrot-server/target/foxtrot-server*.jar server.jar ADD scripts/local_es_setup.sh local_es_setup.sh -CMD sh -c "sleep 15 ; java -jar server.jar initialize docker.yml || true ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" +CMD sh -c "sleep 15 ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" diff --git a/GAUTH b/GAUTH new file mode 100644 index 000000000..9fb9c9b3e --- /dev/null +++ b/GAUTH @@ -0,0 +1,3 @@ + 280204865700-2hlhc09kfd7s5efek730pq515cn1q3p9.apps.googleusercontent.com + + 6dy_KawiZLdaX--SOH2xX4_y diff --git a/config/docker.yml b/config/docker.yml index 6e6b76ff2..8f515f1c7 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -82,8 +82,13 @@ auth: sessionDuration: 15d provider: - type: OAUTH_GOOGLE - clientId: ${GOOGLE_CLIENT_ID} - clientSecret: ${GOOGLE_CLIENT_SECRET} - server: ${GOOGLE_CALLBACK_HOST_PORT} - secureEndpoint: ${GOOGLE_SECURE_ENDPOINT} +# type: OAUTH_GOOGLE +# clientId: ${GOOGLE_CLIENT_ID} +# clientSecret: ${GOOGLE_CLIENT_SECRET} +# server: ${GOOGLE_CALLBACK_HOST_PORT} +# secureEndpoint: ${GOOGLE_SECURE_ENDPOINT} + type: OAUTH_IDMAN + idmanEndpoint: https://ssidman.loca.lt + clientId: FOXTROT + clientSecret: 8f4bcf45-2909-42e0-9bba-f46351bb0e6d + serverEndpoint: "http://localhost:17000" diff --git a/docker-compose.yml b/docker-compose.yml index fa57ea60f..0b9544f73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,20 +15,20 @@ services: command: elasticsearch -Ehttp.cors.enabled=true volumes: - /elasticsearch/data - ports: - - "9200:9200" - - "9300:9300" +# ports: +# - "9200:9200" +# - "9300:9300" hbase: image: hyness/hbase-rest-standalone hostname: hbase - ports: - - "2181:2181" - - "8080:8080" - - "16000:16000" - - "16010:16010" - - "16020:16020" - - "16030:16030" +# ports: +# - "2181:2181" +# - "8080:8080" +# - "16000:16000" +# - "16010:16010" +# - "16020:16020" +# - "16030:16030" volumes: - /data/hbase - /data/zookeeper @@ -57,7 +57,7 @@ services: - HBASE_ZOOKEEPER_QUORUM=hbase - HBASE_ZOOKEEPER_CLIENT_PORT=2181 - HAZELCAST_CLUSTER_NAME=foxtrot - - AUTH_ENABLED=false + - AUTH_ENABLED=true - GOOGLE_CALLBACK_HOST_PORT=localhost:17000 - GOOGLE_CLIENT_ID=dummy - GOOGLE_CLIENT_SECRET=dummy diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 2ed56f1a1..dd05ab1b8 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -96,6 +96,17 @@ google-api-client 1.23.0 + + io.appform.idman + idman-model + 1.0-SNAPSHOT + + + * + * + + + com.hazelcast hazelcast diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/AuthInfoType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/AuthInfoType.java new file mode 100644 index 000000000..cbeefc2f1 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/AuthInfoType.java @@ -0,0 +1,8 @@ +package com.flipkart.foxtrot.server; + +/** + * + */ +public enum AuthInfoType { + TOKEN +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfo.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfo.java new file mode 100644 index 000000000..ecee575af --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfo.java @@ -0,0 +1,18 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.server.AuthInfoType; +import lombok.Data; + +/** + * + */ +@Data +public abstract class AuthInfo { + private final AuthInfoType type; + + protected AuthInfo(AuthInfoType type) { + this.type = type; + } + + public abstract T accept(final AuthInfoVisitor visitor); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfoVisitor.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfoVisitor.java new file mode 100644 index 000000000..91dc17a6a --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfoVisitor.java @@ -0,0 +1,8 @@ +package com.flipkart.foxtrot.server.auth; + +/** + * + */ +public interface AuthInfoVisitor { + T visit(TokenAuthInfo tokenAuthInfo); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthenticatedInfo.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthenticatedInfo.java new file mode 100644 index 000000000..5427ed187 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthenticatedInfo.java @@ -0,0 +1,13 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.User; +import lombok.Value; + +/** + * + */ +@Value +public class AuthenticatedInfo { + Token token; + User user; +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java index f897d5246..57d2242c7 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -4,6 +4,7 @@ import com.flipkart.foxtrot.core.auth.User; import com.flipkart.foxtrot.core.querystore.actions.Utils; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import com.flipkart.foxtrot.server.auth.authprovider.IdType; import io.dropwizard.util.Duration; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -112,7 +113,7 @@ public Optional provisionToken(String userId, String tokenId, TokenType t try { val saveStatus = connection.getClient() .index(new IndexRequest(TOKENS_INDEX) - .source(mapper.writeValueAsString(new Token(tokenId, tokenType, userId, expiry)), + .source(mapper.writeValueAsString(new Token(tokenId, IdType.SESSION_ID, tokenType, userId, expiry)), XContentType.JSON) .id(tokenId) .type(TOKEN_TYPE) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java new file mode 100644 index 000000000..9a9e27ad2 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java @@ -0,0 +1,86 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import io.dropwizard.util.Duration; + +import java.util.Date; +import java.util.Optional; +import java.util.function.UnaryOperator; + +/** + * + */ +public class IdmanAuthStore implements AuthStore { + @Override + public Optional provisionUser(User user) { + return Optional.empty(); + } + + @Override + public Optional getUser(String userId) { + return Optional.empty(); + } + + @Override + public boolean deleteUser(String id) { + return false; + } + + @Override + public boolean updateUser( + String id, UnaryOperator mutator) { + return false; + } + + @Override + public boolean grantRole(String userId, FoxtrotRole role) { + return AuthStore.super.grantRole(userId, role); + } + + @Override + public boolean revokeRole(String userId, FoxtrotRole role) { + return AuthStore.super.revokeRole(userId, role); + } + + @Override + public boolean grantTableAccess(String userId, String table) { + return AuthStore.super.grantTableAccess(userId, table); + } + + @Override + public boolean revokeTableAccess(String userId, String table) { + return AuthStore.super.revokeTableAccess(userId, table); + } + + @Override + public Optional provisionToken(String userId, TokenType tokenType, Date expiry) { + return AuthStore.super.provisionToken(userId, tokenType, expiry); + } + + @Override + public Optional provisionToken( + String userId, String tokenId, TokenType tokenType, Date expiry) { + return Optional.empty(); + } + + @Override + public Optional getToken(String tokenId) { + return Optional.empty(); + } + + @Override + public Optional getTokenForUser(String userId) { + return Optional.empty(); + } + + @Override + public boolean deleteToken(String tokenId) { + return false; + } + + @Override + public boolean deleteExpiredTokens(Date date, Duration sessionDuration) { + return false; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java index 1ffb56288..e7cab5300 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java @@ -1,5 +1,6 @@ package com.flipkart.foxtrot.server.auth; +import com.flipkart.foxtrot.server.auth.authprovider.IdType; import lombok.Value; import java.util.Date; @@ -9,9 +10,14 @@ */ @Value public class Token { - public static final Token DEFAULT = new Token("__DEFAULT_TOKEN__", TokenType.SYSTEM, "__DEFAULT__", null); + public static final Token DEFAULT = new Token("__DEFAULT_TOKEN__", + IdType.SESSION_ID, + TokenType.SYSTEM, + "__DEFAULT__", + null); String id; + IdType idType; TokenType tokenType; String userId; Date expiry; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthInfo.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthInfo.java new file mode 100644 index 000000000..a3caeca7e --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthInfo.java @@ -0,0 +1,27 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.server.AuthInfoType; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import lombok.Value; + +/** + * + */ +@Value +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class TokenAuthInfo extends AuthInfo { + + String token; + + public TokenAuthInfo(String token) { + super(AuthInfoType.TOKEN); + this.token = token; + } + + @Override + public T accept(AuthInfoVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java index b2931cf1f..2416ef119 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java @@ -1,11 +1,11 @@ package com.flipkart.foxtrot.server.auth; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.google.common.base.Strings; import io.dropwizard.auth.AuthenticationException; import io.dropwizard.auth.Authenticator; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.jose4j.jwt.MalformedClaimException; -import org.jose4j.jwt.consumer.JwtContext; import javax.inject.Inject; import javax.inject.Provider; @@ -17,50 +17,35 @@ */ @Slf4j @Singleton -public class TokenAuthenticator implements Authenticator { +public class TokenAuthenticator implements Authenticator { private final AuthConfig config; - private final Provider authStore; + private final Provider provider; + @Inject - public TokenAuthenticator(AuthConfig config, final Provider authStore) { + public TokenAuthenticator(AuthConfig config, Provider provider) { this.config = config; - this.authStore = authStore; + this.provider = provider; } @Override - public Optional authenticate(JwtContext jwtContext) throws AuthenticationException { + public Optional authenticate(String token) throws AuthenticationException { if(!config.isEnabled()) { log.debug("Authentication is disabled"); return Optional.of(UserPrincipal.DEFAULT); } log.debug("Auth called"); - final String userId; - final String tokenId; - try { - val claims = jwtContext.getJwtClaims(); - userId = claims.getSubject(); - tokenId = claims.getJwtId(); - } - catch (MalformedClaimException e) { - log.error(String.format("exception in claim extraction %s", e.getMessage()), e); - return Optional.empty(); - } - log.debug("authentication_requested userId:{} tokenId:{}", userId, tokenId); - val token = authStore.get().getToken(tokenId).orElse(null); - if (token == null) { - log.warn("authentication_failed::invalid_session userId:{} tokenId:{}", userId, tokenId); - return Optional.empty(); - } - if (!token.getUserId().equals(userId)) { - log.warn("authentication_failed::user_mismatch userId:{} tokenId:{}", userId, tokenId); + if(Strings.isNullOrEmpty(token)) { + log.warn("authentication_failed::empty token"); return Optional.empty(); } - val user = authStore.get().getUser(token.getUserId()).orElse(null); - if (null == user) { - log.warn("authentication_failed::invalid_user userId:{} tokenId:{}", userId, tokenId); + val info = provider.get() + .authenticate(new TokenAuthInfo(token)) + .orElse(null); + if (info == null) { + log.warn("authentication_failed::token_validation_failed"); return Optional.empty(); } - log.debug("authentication_success userId:{} tokenId:{}", userId, tokenId); - return Optional.of(new UserPrincipal(user, token)); + return Optional.of(new UserPrincipal(info.getUser(), info.getToken())); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java index ddf5b4886..41dac7f9a 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java @@ -1,10 +1,13 @@ package com.flipkart.foxtrot.server.auth.authprovider; import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProviderConfig; /** * */ public interface AuthConfigVisitor { T visit(GoogleAuthProviderConfig googleAuthConfig); + + T visit(IdmanAuthProviderConfig idmanAuthProviderConfig); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java index 8d526bbbc..48cfecb3b 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java @@ -1,8 +1,8 @@ package com.flipkart.foxtrot.server.auth.authprovider; +import com.flipkart.foxtrot.server.auth.AuthInfo; +import com.flipkart.foxtrot.server.auth.AuthenticatedInfo; import com.flipkart.foxtrot.server.auth.Token; -import lombok.Builder; -import lombok.Data; import java.util.Optional; @@ -11,14 +11,6 @@ */ public interface AuthProvider { - - @Data - @Builder - public class AuthInfo { - private final String ip; - private final String email; - } - AuthType type(); String redirectionURL(String sessionId); @@ -28,4 +20,6 @@ default boolean isPreregistrationRequired() { } Optional login(String authCode, String sessionId); + + Optional authenticate(AuthInfo authInfo); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java index 97d918eb5..136d6e4f7 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProviderConfig; import lombok.Data; /** @@ -11,6 +12,7 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type") @JsonSubTypes({ @JsonSubTypes.Type(name = "OAUTH_GOOGLE", value = GoogleAuthProviderConfig.class), + @JsonSubTypes.Type(name = "OAUTH_IDMAN", value = IdmanAuthProviderConfig.class), }) @Data public abstract class AuthProviderConfig { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java index 9435b56e4..4dc8d4648 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java @@ -1,10 +1,13 @@ package com.flipkart.foxtrot.server.auth.authprovider; -import java.util.Map; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.google.inject.Injector; /** * */ public interface AuthProviderFactory { - Map build(); + + AuthProvider build(ObjectMapper mapper, AuthStore authStore); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java index 71e18aebe..4e3d660d2 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java @@ -4,5 +4,6 @@ * */ public enum AuthType { - OAUTH_GOOGLE + OAUTH_GOOGLE, + OAUTH_IDMAN } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java new file mode 100644 index 000000000..7ede8768b --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java @@ -0,0 +1,36 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProviderConfig; + +/** + * + */ +public class ConfiguredAuthProviderFactory implements AuthProviderFactory { + private final AuthConfig authConfig; + + public ConfiguredAuthProviderFactory(AuthConfig authConfig) { + this.authConfig = authConfig; + } + + @Override + public AuthProvider build(ObjectMapper mapper, AuthStore authStore) { + return authConfig.getProvider() + .accept(new AuthConfigVisitor() { + @Override + public AuthProvider visit(GoogleAuthProviderConfig googleAuthConfig) { + return new GoogleAuthProvider(googleAuthConfig, authConfig, mapper, authStore); + } + + @Override + public AuthProvider visit(IdmanAuthProviderConfig idmanAuthProviderConfig) { + return new IdmanAuthProvider(idmanAuthProviderConfig, mapper); + } + }); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/IdType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/IdType.java new file mode 100644 index 000000000..dcb06a85a --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/IdType.java @@ -0,0 +1,9 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +/** + * + */ +public enum IdType { + SESSION_ID, + ACCESS_TOKEN +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java index d2ae5a5f2..3ca298544 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java @@ -1,16 +1,12 @@ package com.flipkart.foxtrot.server.auth.authprovider.impl; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.foxtrot.server.auth.AuthConfig; -import com.flipkart.foxtrot.server.auth.AuthStore; -import com.flipkart.foxtrot.server.auth.Token; -import com.flipkart.foxtrot.server.auth.TokenType; +import com.flipkart.foxtrot.server.auth.*; import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; import com.flipkart.foxtrot.server.auth.authprovider.AuthType; import com.flipkart.foxtrot.server.utils.AuthUtils; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeTokenRequest; import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; @@ -25,10 +21,20 @@ import io.dropwizard.util.Duration; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.jose4j.jwa.AlgorithmConstraints; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jwt.MalformedClaimException; +import org.jose4j.jwt.consumer.InvalidJwtException; +import org.jose4j.jwt.consumer.JwtConsumer; +import org.jose4j.jwt.consumer.JwtConsumerBuilder; +import org.jose4j.jwt.consumer.JwtContext; +import org.jose4j.keys.HmacKey; import java.io.IOException; import java.net.InetSocketAddress; import java.net.Proxy; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Date; import java.util.Optional; @@ -38,7 +44,7 @@ @Slf4j public class GoogleAuthProvider implements AuthProvider { - public static final String CALLBACK_PATH = "/foxtrot/google/callback"; + public static final String CALLBACK_PATH = "/foxtrot/oauth/callback"; private final HttpTransport transport; private final GoogleAuthorizationCodeFlow authorizationCodeFlow; @@ -47,6 +53,7 @@ public class GoogleAuthProvider implements AuthProvider { private final GoogleAuthProviderConfig googleAuthConfig; private final ObjectMapper mapper; private final AuthStore credentialsStorage; + private final JwtConsumer consumer; @Inject public GoogleAuthProvider( @@ -93,6 +100,7 @@ public GoogleAuthProvider( this.googleAuthConfig = googleAuthConfig; this.mapper = mapper; this.credentialsStorage = credentialsStorage; + this.consumer = buildJwtConsumer(); } @Override @@ -108,8 +116,8 @@ public String redirectionURL(String sessionId) { // .setRedirectUri("http://localhost:8080/auth/google") .build(); return !Strings.isNullOrEmpty(googleAuthConfig.getLoginDomain()) - ? (url + "&hd=" + googleAuthConfig.getLoginDomain()) - : url; + ? (url + "&hd=" + googleAuthConfig.getLoginDomain()) + : url; } @Override @@ -117,7 +125,7 @@ public Optional login(String authToken, String sessionId) { if (Strings.isNullOrEmpty(authToken)) { return Optional.empty(); } - final GoogleAuthorizationCodeTokenRequest googleAuthorizationCodeTokenRequest + val googleAuthorizationCodeTokenRequest = authorizationCodeFlow.newTokenRequest(authToken); final String email; try { @@ -151,9 +159,68 @@ public Optional login(String authToken, String sessionId) { new Date(new Date().getTime() + sessionDuration.toMilliseconds())); } + @Override + public Optional authenticate(AuthInfo authInfo) { + val jwt = authInfo.accept(TokenAuthInfo::getToken); + final JwtContext jwtContext; + try { + jwtContext = consumer.process(jwt); + } + catch (InvalidJwtException e) { + log.error("Jwt validation failure: ", e); + return Optional.empty(); + } + final String userId; + final String tokenId; + try { + val claims = jwtContext.getJwtClaims(); + userId = claims.getSubject(); + tokenId = claims.getJwtId(); + } + catch (MalformedClaimException e) { + log.error(String.format("exception in claim extraction %s", e.getMessage()), e); + return Optional.empty(); + } + log.debug("authentication_requested userId:{} tokenId:{}", userId, tokenId); + val token = credentialsStorage.getToken(tokenId).orElse(null); + if (token == null) { + log.warn("authentication_failed::invalid_session userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + if (!token.getUserId().equals(userId)) { + log.warn("authentication_failed::user_mismatch userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + val user = credentialsStorage.getUser(token.getUserId()).orElse(null); + if (null == user) { + log.warn("authentication_failed::invalid_user userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + log.debug("authentication_success userId:{} tokenId:{}", userId, tokenId); + + return Optional.of(new AuthenticatedInfo(token, user)); + } + @Override public boolean isPreregistrationRequired() { return false; } + private JwtConsumer buildJwtConsumer() { + final JwtConfig jwtConfig = authConfig.getJwt(); + final byte[] secretKey = jwtConfig.getPrivateKey().getBytes(StandardCharsets.UTF_8); + return new JwtConsumerBuilder() + .setRequireIssuedAt() + .setRequireSubject() + .setExpectedIssuer(jwtConfig.getIssuerId()) + .setVerificationKey(new HmacKey(secretKey)) + .setJwsAlgorithmConstraints(new AlgorithmConstraints( + AlgorithmConstraints.ConstraintType.WHITELIST, + AlgorithmIdentifiers.HMAC_SHA512)) + .setExpectedAudience(Arrays.stream(TokenType.values()) + .map(TokenType::name) + .toArray(String[]::new)) + .build(); + + } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java new file mode 100644 index 000000000..bce03fd64 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java @@ -0,0 +1,157 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.server.auth.*; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; +import com.flipkart.foxtrot.server.auth.authprovider.IdType; +import com.google.common.collect.ImmutableList; +import io.appform.idman.model.IdmanUser; +import io.appform.idman.model.TokenInfo; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.time.Instant; +import java.util.*; +import java.util.function.Function; + +/** + * + */ +@Slf4j +public class IdmanAuthProvider implements AuthProvider { + private final CloseableHttpClient client = HttpClients.createDefault(); + private final IdmanAuthProviderConfig config; + private final ObjectMapper mapper; + + public IdmanAuthProvider(IdmanAuthProviderConfig config, ObjectMapper mapper) { + this.config = config; + this.mapper = mapper; + + } + + @Override + public AuthType type() { + return AuthType.OAUTH_IDMAN; + } + + @Override + @SneakyThrows + public String redirectionURL(String sessionId) { + return new URIBuilder(config.getIdmanEndpoint()) + .setPath("/apis/oauth2/authorize") + .addParameter("response_type", "code") + .addParameter("client_id", config.getClientId()) + .addParameter("state", sessionId) + .addParameter("redirect_uri", config.getServerEndpoint() + "/foxtrot/oauth/callback") + .build() + .toString(); + } + + @Override + @SneakyThrows + public Optional login(String authCode, String sessionId) { + return callIdmanServer("authorization_code", authCode, + ti -> { + val idmanUser = ti.getUser(); + val expiry = Date.from(Instant.now().plusSeconds(ti.getExpiry())); + return Optional.of(new Token(ti.getAccessToken(), + IdType.ACCESS_TOKEN, + TokenType.DYNAMIC, + idmanUser.getUser().getId(), + expiry)); + }); + } + + @Override + @SneakyThrows + public Optional authenticate(AuthInfo authInfo) { + return refreshToken(authInfo.accept(TokenAuthInfo::getToken)); + } + + @SneakyThrows + private Optional refreshToken(String token) { + return callIdmanServer( + "refresh_token", + token, + ti -> { + val idmanUser = ti.getUser(); + val expiry = Date.from(Instant.now().plusSeconds(ti.getExpiry())); + val remoteUser = idmanUser.getUser(); + val t = new Token(ti.getAccessToken(), + IdType.ACCESS_TOKEN, + TokenType.DYNAMIC, + remoteUser.getId(), + expiry); + val user = new User(remoteUser.getId(), + mapToFoxtrotRoles(idmanUser), + Collections.emptySet(), + isSystemUser(idmanUser), + new Date(), + new Date()); + return Optional.of(new AuthenticatedInfo(t, user)); + }); + } + + private Set mapToFoxtrotRoles(final IdmanUser idmanUser) { + switch (idmanUser.getRole()) { + case "FOXTROT_ADMIN": { + return EnumSet.allOf(FoxtrotRole.class); + } + case "FOXTROT_HUMAN_USER": { + return EnumSet.of(FoxtrotRole.CONSOLE, FoxtrotRole.QUERY, FoxtrotRole.INGEST); + } + case "FOXTROT_SYSTEM_USER": { + return EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.INGEST); + } + default: + throw new IllegalStateException("Unexpected value: " + idmanUser.getRole()); + } + } + + private boolean isSystemUser(final IdmanUser user) { + return user.getRole().equals("FOXTROT_ADMIN"); + } + + private Optional callIdmanServer( + String callMode, + String param, + Function> handler) throws URISyntaxException, IOException { + val post = new HttpPost(new URIBuilder(config.getIdmanEndpoint()) + .setPath("/apis/oauth2/token") + .build()); + val fromBuilder = ImmutableList.builder(); + val form = fromBuilder + .add(new BasicNameValuePair(callMode.equals("authorization_code") + ? "code" + : "refresh_token", param)) + .add(new BasicNameValuePair("client_id", config.getClientId())) + .add(new BasicNameValuePair("client_secret", config.getClientSecret())) + .add(new BasicNameValuePair("grant_type", callMode)) + .build(); + post.setEntity(new UrlEncodedFormEntity(form)); + try (val response = client.execute(post)) { + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + log.error("Error calling {}: {}", post.getURI().toString(), response.getStatusLine()); + return Optional.empty(); + } + val ti = mapper.readValue(EntityUtils.toByteArray(response.getEntity()), TokenInfo.class); + return handler.apply(ti); + } + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProviderConfig.java new file mode 100644 index 000000000..f16c9ea93 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProviderConfig.java @@ -0,0 +1,49 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.flipkart.foxtrot.server.auth.authprovider.AuthConfigVisitor; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.hibernate.validator.constraints.NotEmpty; + +/** + * + */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class IdmanAuthProviderConfig extends AuthProviderConfig { + @NotEmpty + private String idmanEndpoint; + @NotEmpty + private String clientId; + @NotEmpty + private String clientSecret; + @NotEmpty + public String serverEndpoint; + + public IdmanAuthProviderConfig() { + super(AuthType.OAUTH_IDMAN); + } + + @Builder + public IdmanAuthProviderConfig( + AuthType type, + boolean enabled, + String idmanEndpoint, + String clientId, + String clientSecret) { + super(type, enabled); + this.idmanEndpoint = idmanEndpoint; + this.clientId = clientId; + this.clientSecret = clientSecret; + } + + @Override + public T accept(AuthConfigVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java index f417728ff..2c0c2426f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java @@ -2,16 +2,12 @@ import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.server.auth.UserPrincipal; -import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import io.dropwizard.auth.AuthenticationException; import io.dropwizard.auth.Authenticator; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.jose4j.jwt.consumer.InvalidJwtException; -import org.jose4j.jwt.consumer.JwtConsumer; -import org.jose4j.jwt.consumer.JwtContext; import javax.annotation.Priority; import javax.inject.Inject; @@ -36,23 +32,16 @@ @Slf4j public class UserAuthenticationFilter implements Filter { private static final Set WHITELISTED_PATTERNS = ImmutableSet.builder() - .add("/foxtrot/google") + .add("/foxtrot/oauth") .add("^/foxtrot/auth.*") .build(); private final AuthConfig authConfig; - private final Provider sessionDataStore; - private final JwtConsumer consumer; - private final Provider> authenticator; + private final Provider> authenticator; @Inject public UserAuthenticationFilter( - AuthConfig authConfig, - Provider sessionDataStore, - JwtConsumer consumer, - Provider> authenticator) { + AuthConfig authConfig, Provider> authenticator) { this.authConfig = authConfig; - this.sessionDataStore = sessionDataStore; - this.consumer = consumer; this.authenticator = authenticator; } @@ -77,25 +66,24 @@ public void doFilter( return; } val jwt = getTokenFromCookieOrHeader(httpRequest).orElse(null); - if(null != jwt) { + if(!Strings.isNullOrEmpty(jwt)) { try { - final JwtContext context = consumer.process(jwt); val principal = authenticator.get() - .authenticate(context).orElse(null); + .authenticate(jwt).orElse(null); if(null != principal) { SessionUser.put(principal); chain.doFilter(request, response); return; } } - catch (InvalidJwtException | AuthenticationException e) { + catch (AuthenticationException e) { log.error("Jwt validation failure: ", e); } } val referrer = httpRequest.getHeader(org.apache.http.HttpHeaders.REFERER); val source = Strings.isNullOrEmpty(referrer) ? requestURI : referrer; httpResponse.addCookie(new Cookie("redirection", source)); - httpResponse.sendRedirect("/foxtrot/google/login"); + httpResponse.sendRedirect("/foxtrot/oauth/login"); } @Override diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java index 9676989b9..68ec99d8f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java @@ -37,8 +37,7 @@ import com.flipkart.foxtrot.core.table.impl.FoxtrotTableManager; import com.flipkart.foxtrot.server.auth.*; import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; -import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProvider; -import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.ConfiguredAuthProviderFactory; import com.flipkart.foxtrot.server.auth.sessionstore.DistributedSessionDataStore; import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; @@ -53,6 +52,7 @@ import com.google.common.cache.CacheBuilderSpec; import com.google.common.collect.ImmutableList; import com.google.inject.AbstractModule; +import com.google.inject.Injector; import com.google.inject.Provides; import com.google.inject.TypeLiteral; import io.dropwizard.auth.Authenticator; @@ -62,12 +62,12 @@ import io.dropwizard.server.ServerFactory; import io.dropwizard.setup.Environment; import io.dropwizard.util.Duration; +import lombok.val; import org.apache.hadoop.conf.Configuration; import org.jose4j.jwa.AlgorithmConstraints; import org.jose4j.jws.AlgorithmIdentifiers; import org.jose4j.jwt.consumer.JwtConsumer; import org.jose4j.jwt.consumer.JwtConsumerBuilder; -import org.jose4j.jwt.consumer.JwtContext; import org.jose4j.keys.HmacKey; import javax.inject.Singleton; @@ -112,8 +112,6 @@ protected void configure() { bind(new TypeLiteral>() {}).toProvider(HealthcheckListProvider.class); bind(AuthStore.class) .to(ESAuthStore.class); - bind(AuthProvider.class) - .to(GoogleAuthProvider.class); bind(SessionDataStore.class) .to(DistributedSessionDataStore.class); } @@ -261,10 +259,34 @@ public AuthConfig authConfig(FoxtrotServerConfiguration serverConfiguration) { return serverConfiguration.getAuth(); } - @Provides +/* @Provides @Singleton public GoogleAuthProviderConfig googleAuthProviderConfig(FoxtrotServerConfiguration configuration) { return (GoogleAuthProviderConfig)configuration.getAuth().getProvider(); + }*/ + + @Provides + @Singleton + public AuthProvider authProvider( + FoxtrotServerConfiguration configuration, + Environment environment, + Injector injector) { + val authType = configuration.getAuth().getProvider().getType(); + AuthStore authStore = null; + switch (authType) { + + case OAUTH_GOOGLE: + authStore = injector.getInstance(ESAuthStore.class); + break; + case OAUTH_IDMAN: + authStore = injector.getInstance(IdmanAuthStore.class); + break; + default: { + throw new IllegalArgumentException("Mode " + authType.name() + " not supported"); + } + } + return new ConfiguredAuthProviderFactory(configuration.getAuth()) + .build(environment.getObjectMapper(), authStore); } @Provides @@ -288,7 +310,7 @@ public JwtConsumer provideJwtConsumer(AuthConfig config) { @Provides @Singleton - public Authenticator authenticator( + public Authenticator authenticator( final Environment environment, final TokenAuthenticator authenticator, final AuthConfig authConfig) { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java index b936fa4a0..85845391b 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java @@ -20,6 +20,6 @@ public class AuthenticationExceptionHandler implements ExceptionMapper authProvider; private final Provider sessionDataStore; @Inject - public GoogleAuth( + public OAuth( AuthConfig authConfig, Provider authProvider, Provider sessionDataStore) { @@ -46,7 +48,7 @@ public Response login(@CookieParam("redirection") final Cookie cookieReferrer, final String sessionId = UUID.randomUUID().toString(); final String redirectionURL = authProvider.get() .redirectionURL(sessionId); - log.debug("Redirection uri: {}", redirectionURL); + log.info("Redirection uri: {}", redirectionURL); final String cookieReferrerUrl = null == cookieReferrer ? null : cookieReferrer.getValue(); val source = Strings.isNullOrEmpty(cookieReferrerUrl) ? referrer : cookieReferrerUrl; log.debug("Call source: {} Referrer: {} Redirection: {}", source, referrer, cookieReferrerUrl); @@ -62,7 +64,7 @@ public Response login(@CookieParam("redirection") final Cookie cookieReferrer, sessionId, GoogleAuthProvider.CALLBACK_PATH, null, - NewCookie.DEFAULT_VERSION, + Cookie.DEFAULT_VERSION, null, NewCookie.DEFAULT_MAX_AGE, null, @@ -89,7 +91,7 @@ public Response handleGoogleCallback( .login(authCode, sessionId) .orElse(null); if (null == token) { - return Response.seeOther(URI.create("/foxtrot/login/google")).build(); + return Response.seeOther(URI.create("/foxtrot/oauth/login")).build(); } val existingReferrer = sessionDataStore.get() .get(token.getId()) @@ -103,7 +105,9 @@ public Response handleGoogleCallback( log.debug("Will be redirecting to: {}. Existing: {}", finalRedirect, existingReferrer); return Response.seeOther(URI.create(finalRedirect)) .cookie(new NewCookie("token", - AuthUtils.createJWT(token, authConfig.getJwt()), + token.getIdType().equals(IdType.SESSION_ID) + ? AuthUtils.createJWT(token, authConfig.getJwt()) + : token.getId(), "/", null, Cookie.DEFAULT_VERSION, @@ -115,4 +119,13 @@ public Response handleGoogleCallback( new NewCookie(cookieState, null, 0, false)) .build(); } + + @GET + @Path("/logout") + @PermitAll + public Response logout(@CookieParam("token") final Cookie token) { + return Response.seeOther(URI.create("/")) + .cookie(new NewCookie(token, null, 0, false)) + .build(); + } } diff --git a/foxtrot-server/src/main/resources/console/echo/browse-events.htm b/foxtrot-server/src/main/resources/console/echo/browse-events.htm index ca35ed910..0c144a1bb 100644 --- a/foxtrot-server/src/main/resources/console/echo/browse-events.htm +++ b/foxtrot-server/src/main/resources/console/echo/browse-events.htm @@ -66,6 +66,7 @@

    • Create New Dashboard
      • +
    • Logout

    diff --git a/foxtrot-server/src/main/resources/console/echo/fql/index.htm b/foxtrot-server/src/main/resources/console/echo/fql/index.htm index 0951accf6..f0f4d42f0 100644 --- a/foxtrot-server/src/main/resources/console/echo/fql/index.htm +++ b/foxtrot-server/src/main/resources/console/echo/fql/index.htm @@ -92,6 +92,7 @@

  • Create New Dashboard
    • +
  • Logout

diff --git a/foxtrot-server/src/main/resources/console/echo/index.htm b/foxtrot-server/src/main/resources/console/echo/index.htm index 9b0e91ce6..a3c9ff03b 100644 --- a/foxtrot-server/src/main/resources/console/echo/index.htm +++ b/foxtrot-server/src/main/resources/console/echo/index.htm @@ -68,6 +68,7 @@

  • Create New Dashboard
    • +
  • Logout
    • diff --git a/out b/out new file mode 100644 index 000000000..37f79ce33 --- /dev/null +++ b/out @@ -0,0 +1,774 @@ +Attaching to elasticsearch, foxtrot_hbase_1, foxtrot_server +hbase_1 | /usr/lib/python2.7/dist-packages/supervisor/options.py:296: UserWarning: Supervisord is running as root and it is searching for its configuration file in default locations (including its current working directory); you probably want to specify a "-c" argument specifying an absolute path to a configuration file for improved security. +hbase_1 | 'Supervisord is running as root and it is searching ' +foxtrot_server | No CONFIG_PATH defined. We shall be using default config from /config/docker.yml +hbase_1 | 2020-07-22 09:39:17,221 CRIT Supervisor running as root (no user in config file) +foxtrot_server | Init sleep of 15 seconds specified. Waiting... +hbase_1 | 2020-07-22 09:39:17,221 WARN Included extra file "/etc/supervisor/conf.d/supervisord.conf" during parsing +hbase_1 | Unlinking stale socket /var/run/supervisor.sock +hbase_1 | 2020-07-22 09:39:17,531 INFO RPC interface 'supervisor' initialized +hbase_1 | 2020-07-22 09:39:17,531 CRIT Server 'unix_http_server' running without any HTTP authentication checking +hbase_1 | 2020-07-22 09:39:17,531 INFO supervisord started with pid 1 +hbase_1 | 2020-07-22 09:39:18,533 INFO spawned: 'stdout' with pid 8 +hbase_1 | 2020-07-22 09:39:18,535 INFO spawned: 'hbase-rest' with pid 9 +hbase_1 | 2020-07-22 09:39:18,536 INFO spawned: 'hbase' with pid 10 +hbase_1 | starting rest, logging to /opt/hbase/bin/../logs/hbase--rest-hbase.out +elasticsearch | [2020-07-22T09:39:19,168][INFO ][o.e.e.NodeEnvironment ] [5q9Ine5] using [1] data paths, mounts [[/ (overlay)]], net usable_space [2.4gb], net total_space [28.5gb], types [overlay] +elasticsearch | [2020-07-22T09:39:19,171][INFO ][o.e.e.NodeEnvironment ] [5q9Ine5] heap size [512mb], compressed ordinary object pointers [true] +elasticsearch | [2020-07-22T09:39:19,200][INFO ][o.e.n.Node ] [5q9Ine5] node name derived from node ID [5q9Ine5JToGzVTBOdMThhw]; set [node.name] to override +elasticsearch | [2020-07-22T09:39:19,200][INFO ][o.e.n.Node ] [5q9Ine5] version[6.8.8], pid[1], build[oss/docker/2f4c224/2020-03-18T23:22:18.622755Z], OS[Linux/5.3.0-62-generic/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/14/14+36] +elasticsearch | [2020-07-22T09:39:19,201][INFO ][o.e.n.Node ] [5q9Ine5] JVM arguments [-Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/tmp/elasticsearch-4753755361850039883, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.locale.providers=COMPAT, -XX:UseAVX=2, -Xms512m, -Xmx512m, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=oss, -Des.distribution.type=docker] +hbase_1 | 2020-07-22 09:39:19,573 INFO success: stdout entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +hbase_1 | 2020-07-22 09:39:19,573 INFO success: hbase-rest entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +hbase_1 | 2020-07-22 09:39:19,573 INFO success: hbase entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +hbase_1 | hbase stderr | OpenJDK 64-Bit Server VM warning: hbase stderr | ignoring option PermSize=128m; support was removed in 8.0 hbase stderr | +hbase_1 | hbase stderr | OpenJDK 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0 +hbase_1 | hbase-rest stdout | starting rest, logging to /opt/hbase/bin/../logs/hbase--rest-hbase.out +hbase_1 | hbase stderr | 2020-07-22 09:39:19,097 INFO [main] util.VersionInfo: HBase 1.2.1 +hbase_1 | hbase stderr | 2020-07-22 09:39:19,098 INFO [main] util.VersionInfo: Source code repository git://asf-dev/home/busbey/projects/hbase revision=8d8a7107dc4ccbf36a92f64675dc60392f85c015 +hbase_1 | 2020-07-22 09:39:19,098 INFO [main] util.VersionInfo: Compiled by busbey on Wed Mar 30 11:19:21 CDT 2016 +hbase_1 | 2020-07-22 09:39:19,098 INFO [main] util.VersionInfo: From source with checksum f4bb4a14bb4e0b72b46f729dae98a772 +hbase_1 | hbase stderr | 2020-07-22 09:39:19,572 INFO [main] master.HMasterCommandLine: Starting a zookeeper cluster +hbase_1 | hbase stderr | 2020-07-22 09:39:19,630 INFO [main] server.ZooKeeperServer: Server environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT +hbase_1 | 2020-07-22 09:39:19,634 INFO exited: hbase-rest (exit status 0; expected) +hbase_1 | hbase stderr | 2020-07-22 09:39:19,634 INFO [main] server.ZooKeeperServer: Server environment:host.name=hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:19,638 INFO [main] server.ZooKeeperServer: Server environment:java.version=1.8.0_72-internal +hbase_1 | hbase stderr | 2020-07-22 09:39:19,639 INFO [main] server.ZooKeeperServer: Server environment:java.vendor=Oracle Corporation +hbase_1 | hbase stderr | 2020-07-22 09:39:19,639 INFO [main] server.ZooKeeperServer: Server environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre +hbase_1 | hbase stderr | 2020-07-22 09:39:19,640 INFO [main] server.ZooKeeperServer: Server environment:java.class.path=/opt/hbase/bin/../conf:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../lib/activation-1.1.jar:/opt/hbase/bin/../lib/antisamy-1.4.3.jar:/opt/hbase/bin/../lib/aopalliance-1.0.jar:/opt/hbase/bin/../lib/apacheds-i18n-2.0.0-M15.jar:/opt/hbase/bin/../lib/apacheds-kerberos-codec-2.0.0-M15.jar:/opt/hbase/bin/../lib/api-asn1-api-1.0.0-M20.jar:/opt/hbase/bin/../lib/api-util-1.0.0-M20.jar:/opt/hbase/bin/../lib/asm-3.1.jar:/opt/hbase/bin/../lib/avro-1.7.4.jar:/opt/hbase/bin/../lib/batik-css-1.7.jar:/opt/hbase/bin/../lib/batik-ext-1.7.jar:/opt/hbase/bin/../lib/batik-util-1.7.jar:/opt/hbase/bin/../lib/bsh-core-2.0b4.jar:/opt/hbase/bin/../lib/commons-beanutils-1.7.0.jar:/opt/hbase/bin/../lib/commons-beanutils-core-1.7.0.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.9.jar:/opt/hbase/bin/../lib/commons-collections-3.2.2.jar:/opt/hbase/bin/../lib/commons-compress-1.4.1.jar:/opt/hbase/bin/../lib/commons-configuration-1.6.jar:/opt/hbase/bin/../lib/commons-daemon-1.0.13.jar:/opt/hbase/bin/../lib/commons-digester-1.8.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-fileupload-1.2.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-io-2.4.jar:/opt/hbase/bin/../lib/commons-lang-2.6.jar:/opt/hbase/bin/../lib/commons-logging-1.2.jar:/opt/hbase/bin/../lib/commons-math-2.2.jar:/opt/hbase/bin/../lib/commons-math3-3.1.1.jar:/opt/hbase/bin/../lib/commons-net-3.1.jar:/opt/hbase/bin/../lib/disruptor-3.3.0.jar:/opt/hbase/bin/../lib/esapi-2.1.0.jar:/opt/hbase/bin/../lib/findbugs-annotations-1.3.9-1.jar:/opt/hbase/bin/../lib/guava-12.0.1.jar:/opt/hbase/bin/../lib/guice-3.0.jar:/opt/hbase/bin/../lib/guice-servlet-3.0.jar:/opt/hbase/bin/../lib/hadoop-annotations-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-auth-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-client-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-hdfs-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-app-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-core-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-jobclient-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-shuffle-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-api-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-client-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-server-common-2.5.1.jar:/opt/hbase/bin/../lib/hbase-annotations-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-annotations-1.2.1.jar:/opt/hbase/bin/../lib/hbase-client-1.2.1.jar:/opt/hbase/bin/../lib/hbase-common-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-common-1.2.1.jar:/opt/hbase/bin/../lib/hbase-examples-1.2.1.jar:/opt/hbase/bin/../lib/hbase-external-blockcache-1.2.1.jar:/opt/hbase/bin/../lib/hbase-hadoop-compat-1.2.1.jar:/opt/hbase/bin/../lib/hbase-hadoop2-compat-1.2.1.jar:/opt/hbase/bin/../lib/hbase-it-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-it-1.2.1.jar:/opt/hbase/bin/../lib/hbase-prefix-tree-1.2.1.jar:/opt/hbase/bin/../lib/hbase-procedure-1.2.1.jar:/opt/hbase/bin/../lib/hbase-protocol-1.2.1.jar:/opt/hbase/bin/../lib/hbase-resource-bundle-1.2.1.jar:/opt/hbase/bin/../lib/hbase-rest-1.2.1.jar:/opt/hbase/bin/../lib/hbase-server-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-server-1.2.1.jar:/opt/hbase/bin/../lib/hbase-shell-1.2.1.jar:/opt/hbase/bin/../lib/hbase-thrift-1.2.1.jar:/opt/hbase/bin/../lib/htrace-core-3.1.0-incubating.jar:/opt/hbase/bin/../lib/httpclient-4.2.5.jar:/opt/hbase/bin/../lib/httpcore-4.4.1.jar:/opt/hbase/bin/../lib/jackson-core-asl-1.9.13.jar:/opt/hbase/bin/../lib/jackson-jaxrs-1.9.13.jar:/opt/hbase/bin/../lib/jackson-mapper-asl-1.9.13.jar:/opt/hbase/bin/../lib/jackson-xc-1.9.13.jar:/opt/hbase/bin/../lib/jamon-runtime-2.4.1.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/java-xmlbuilder-0.4.jar:/opt/hbase/bin/../lib/javax.inject-1.jar:/opt/hbase/bin/../lib/jaxb-api-2.2.2.jar:/opt/hbase/bin/../lib/jaxb-impl-2.2.3-1.jar:/opt/hbase/bin/../lib/jcodings-1.0.8.jar:/opt/hbase/bin/../lib/jersey-client-1.9.jar:/opt/hbase/bin/../lib/jersey-core-1.9.jar:/opt/hbase/bin/../lib/jersey-guice-1.9.jar:/opt/hbase/bin/../lib/jersey-json-1.9.jar:/opt/hbase/bin/../lib/jersey-server-1.9.jar:/opt/hbase/bin/../lib/jets3t-0.9.0.jar:/opt/hbase/bin/../lib/jettison-1.3.3.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-sslengine-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/joni-2.1.2.jar:/opt/hbase/bin/../lib/jruby-complete-1.6.8.jar:/opt/hbase/bin/../lib/jsch-0.1.42.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr305-1.3.9.jar:/opt/hbase/bin/../lib/junit-4.12.jar:/opt/hbase/bin/../lib/leveldbjni-all-1.8.jar:/opt/hbase/bin/../lib/libthrift-0.9.3.jar:/opt/hbase/bin/../lib/log4j-1.2.17.jar:/opt/hbase/bin/../lib/metrics-core-2.2.0.jar:/opt/hbase/bin/../lib/nekohtml-1.9.12.jar:/opt/hbase/bin/../lib/netty-all-4.0.23.Final.jar:/opt/hbase/bin/../lib/paranamer-2.3.jar:/opt/hbase/bin/../lib/protobuf-java-2.5.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/servlet-api-2.5.jar:/opt/hbase/bin/../lib/slf4j-api-1.7.7.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.7.5.jar:/opt/hbase/bin/../lib/snappy-java-1.0.4.1.jar:/opt/hbase/bin/../lib/spymemcached-2.11.6.jar:/opt/hbase/bin/../lib/xalan-2.7.0.jar:/opt/hbase/bin/../lib/xml-apis-1.3.03.jar:/opt/hbase/bin/../lib/xml-apis-ext-1.3.04.jar:/opt/hbase/bin/../lib/xmlenc-0.52.jar:/opt/hbase/bin/../lib/xom-1.2.5.jar:/opt/hbase/bin/../lib/xz-1.0.jar:/opt/hbase/bin/../lib/zookeeper-3.4.6.jar: +hbase_1 | hbase stderr | 2020-07-22 09:39:19,640 INFO [main] server.ZooKeeperServer: Server environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib +hbase_1 | hbase stderr | 2020-07-22 09:39:19,641 INFO [main] server.ZooKeeperServer: Server environment:java.io.tmpdir=/tmp +hbase_1 | hbase stderr | 2020-07-22 09:39:19,641 INFO [main] server.ZooKeeperServer: Server environment:java.compiler= +hbase_1 | hbase stderr | 2020-07-22 09:39:19,642 INFO [main] server.ZooKeeperServer: Server environment:os.name=Linux +hbase_1 | hbase stderr | 2020-07-22 09:39:19,642 INFO [main] server.ZooKeeperServer: Server environment:os.arch=amd64 +hbase_1 | hbase stderr | 2020-07-22 09:39:19,642 INFO [main] server.ZooKeeperServer: Server environment:os.version=5.3.0-62-generic +hbase_1 | hbase stderr | 2020-07-22 09:39:19,643 INFO [main] server.ZooKeeperServer: Server environment:user.name=root +hbase_1 | hbase stderr | 2020-07-22 09:39:19,643 INFO [main] server.ZooKeeperServer: Server environment:user.home=/root +hbase_1 | hbase stderr | 2020-07-22 09:39:19,644 INFO [main] server.ZooKeeperServer: Server environment:user.dir=/opt +hbase_1 | hbase stderr | 2020-07-22 09:39:19,695 INFO [main] server.ZooKeeperServer: Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 datadir /data/zookeeper/zookeeper_0/version-2 snapdir /data/zookeeper/zookeeper_0/version-2 +hbase_1 | hbase stderr | 2020-07-22 09:39:19,712 INFO [main] server.NIOServerCnxnFactory: binding to port 0.0.0.0/0.0.0.0:2181 +hbase_1 | hbase stderr | 2020-07-22 09:39:19,731 INFO [main] persistence.FileSnap: Reading snapshot /data/zookeeper/zookeeper_0/version-2/snapshot.1ec33 +elasticsearch | [2020-07-22T09:39:20,627][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [aggs-matrix-stats] +elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [analysis-common] +elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [ingest-common] +elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [ingest-geoip] +elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [ingest-user-agent] +elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [lang-expression] +elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [lang-mustache] +elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [lang-painless] +elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [mapper-extras] +elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [parent-join] +elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [percolator] +elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [rank-eval] +elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [reindex] +elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [repository-url] +elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [transport-netty4] +elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [tribe] +elasticsearch | [2020-07-22T09:39:20,633][INFO ][o.e.p.PluginsService ] [5q9Ine5] no plugins loaded +hbase_1 | hbase stderr | 2020-07-22 09:39:20,371 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34206 +hbase_1 | hbase stderr | 2020-07-22 09:39:20,383 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxn: Processing stat command from /127.0.0.1:34206 +hbase_1 | hbase stderr | 2020-07-22 09:39:20,395 INFO [Thread-2] server.NIOServerCnxn: Stat command output +hbase_1 | hbase stderr | 2020-07-22 09:39:20,396 INFO [Thread-2] server.NIOServerCnxn: Closed socket connection for client /127.0.0.1:34206 (no session established for client) +hbase_1 | hbase stderr | 2020-07-22 09:39:20,396 INFO [main] zookeeper.MiniZooKeeperCluster: Started MiniZooKeeperCluster and ran successful 'stat' on client port=2181 +hbase_1 | hbase stderr | 2020-07-22 09:39:20,397 INFO [main] master.HMasterCommandLine: Starting up instance of localHBaseCluster; master=1, regionserversCount=1 +hbase_1 | hbase stderr | 2020-07-22 09:39:20,794 WARN [main] util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable +hbase_1 | hbase stderr | 2020-07-22 09:39:21,373 INFO [main] regionserver.RSRpcServices: master/hbase/172.18.0.2:0 server-side HConnection retries=350 +hbase_1 | hbase stderr | 2020-07-22 09:39:21,604 INFO [main] ipc.SimpleRpcScheduler: Using deadline as user call queue, count=3 +hbase_1 | hbase stderr | 2020-07-22 09:39:21,624 INFO [main] ipc.RpcServer: master/hbase/172.18.0.2:0: started 10 reader(s) listening on port=41229 +hbase_1 | hbase stderr | 2020-07-22 09:39:21,694 INFO [main] impl.MetricsConfig: loaded properties from hadoop-metrics2-hbase.properties +hbase_1 | hbase stderr | 2020-07-22 09:39:21,719 INFO [main] impl.MetricsSystemImpl: Scheduled snapshot period at 10 second(s). +hbase_1 | 2020-07-22 09:39:21,719 INFO [main] impl.MetricsSystemImpl: HBase metrics system started +hbase_1 | hbase stderr | 2020-07-22 09:39:21,995 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=master:41229 connecting to ZooKeeper ensemble=localhost:2181 +hbase_1 | hbase stderr | 2020-07-22 09:39:22,000 INFO [main] zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT +hbase_1 | hbase stderr | 2020-07-22 09:39:22,000 INFO [main] zookeeper.ZooKeeper: Client environment:host.name=hbase +hbase_1 | 2020-07-22 09:39:22,000 INFO [main] zookeeper.ZooKeeper: Client environment:java.version=1.8.0_72-internal +hbase_1 | 2020-07-22 09:39:22,000 INFO [main] zookeeper.ZooKeeper: Client environment:java.vendor=Oracle Corporation +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre +hbase_1 | hbase stderr | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.class.path=/opt/hbase/bin/../conf:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../lib/activation-1.1.jar:/opt/hbase/bin/../lib/antisamy-1.4.3.jar:/opt/hbase/bin/../lib/aopalliance-1.0.jar:/opt/hbase/bin/../lib/apacheds-i18n-2.0.0-M15.jar:/opt/hbase/bin/../lib/apacheds-kerberos-codec-2.0.0-M15.jar:/opt/hbase/bin/../lib/api-asn1-api-1.0.0-M20.jar:/opt/hbase/bin/../lib/api-util-1.0.0-M20.jar:/opt/hbase/bin/../lib/asm-3.1.jar:/opt/hbase/bin/../lib/avro-1.7.4.jar:/opt/hbase/bin/../lib/batik-css-1.7.jar:/opt/hbase/bin/../lib/batik-ext-1.7.jar:/opt/hbase/bin/../lib/batik-util-1.7.jar:/opt/hbase/bin/../lib/bsh-core-2.0b4.jar:/opt/hbase/bin/../lib/commons-beanutils-1.7.0.jar:/opt/hbase/bin/../lib/commons-beanutils-core-1.7.0.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.9.jar:/opt/hbase/bin/../lib/commons-collections-3.2.2.jar:/opt/hbase/bin/../lib/commons-compress-1.4.1.jar:/opt/hbase/bin/../lib/commons-configuration-1.6.jar:/opt/hbase/bin/../lib/commons-daemon-1.0.13.jar:/opt/hbase/bin/../lib/commons-digester-1.8.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-fileupload-1.2.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-io-2.4.jar:/opt/hbase/bin/../lib/commons-lang-2.6.jar:/opt/hbase/bin/../lib/commons-logging-1.2.jar:/opt/hbase/bin/../lib/commons-math-2.2.jar:/opt/hbase/bin/../lib/commons-math3-3.1.1.jar:/opt/hbase/bin/../lib/commons-net-3.1.jar:/opt/hbase/bin/../lib/disruptor-3.3.0.jar:/opt/hbase/bin/../lib/esapi-2.1.0.jar:/opt/hbase/bin/../lib/findbugs-annotations-1.3.9-1.jar:/opt/hbase/bin/../lib/guava-12.0.1.jar:/opt/hbase/bin/../lib/guice-3.0.jar:/opt/hbase/bin/../lib/guice-servlet-3.0.jar:/opt/hbase/bin/../lib/hadoop-annotations-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-auth-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-client-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-hdfs-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-app-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-core-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-jobclient-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-shuffle-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-api-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-client-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-server-common-2.5.1.jar:/opt/hbase/bin/../lib/hbase-annotations-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-annotations-1.2.1.jar:/opt/hbase/bin/../lib/hbase-client-1.2.1.jar:/opt/hbase/bin/../lib/hbase-common-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-common-1.2.1.jar:/opt/hbase/bin/../lib/hbase-examples-1.2.1.jar:/opt/hbase/bin/../lib/hbase-external-blockcache-1.2.1.jar:/opt/hbase/bin/../lib/hbase-hadoop-compat-1.2.1.jar:/opt/hbase/bin/../lib/hbase-hadoop2-compat-1.2.1.jar:/opt/hbase/bin/../lib/hbase-it-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-it-1.2.1.jar:/opt/hbase/bin/../lib/hbase-prefix-tree-1.2.1.jar:/opt/hbase/bin/../lib/hbase-procedure-1.2.1.jar:/opt/hbase/bin/../lib/hbase-protocol-1.2.1.jar:/opt/hbase/bin/../lib/hbase-resource-bundle-1.2.1.jar:/opt/hbase/bin/../lib/hbase-rest-1.2.1.jar:/opt/hbase/bin/../lib/hbase-server-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-server-1.2.1.jar:/opt/hbase/bin/../lib/hbase-shell-1.2.1.jar:/opt/hbase/bin/../lib/hbase-thrift-1.2.1.jar:/opt/hbase/bin/../lib/htrace-core-3.1.0-incubating.jar:/opt/hbase/bin/../lib/httpclient-4.2.5.jar:/opt/hbase/bin/../lib/httpcore-4.4.1.jar:/opt/hbase/bin/../lib/jackson-core-asl-1.9.13.jar:/opt/hbase/bin/../lib/jackson-jaxrs-1.9.13.jar:/opt/hbase/bin/../lib/jackson-mapper-asl-1.9.13.jar:/opt/hbase/bin/../lib/jackson-xc-1.9.13.jar:/opt/hbase/bin/../lib/jamon-runtime-2.4.1.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/java-xmlbuilder-0.4.jar:/opt/hbase/bin/../lib/javax.inject-1.jar:/opt/hbase/bin/../lib/jaxb-api-2.2.2.jar:/opt/hbase/bin/../lib/jaxb-impl-2.2.3-1.jar:/opt/hbase/bin/../lib/jcodings-1.0.8.jar:/opt/hbase/bin/../lib/jersey-client-1.9.jar:/opt/hbase/bin/../lib/jersey-core-1.9.jar:/opt/hbase/bin/../lib/jersey-guice-1.9.jar:/opt/hbase/bin/../lib/jersey-json-1.9.jar:/opt/hbase/bin/../lib/jersey-server-1.9.jar:/opt/hbase/bin/../lib/jets3t-0.9.0.jar:/opt/hbase/bin/../lib/jettison-1.3.3.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-sslengine-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/joni-2.1.2.jar:/opt/hbase/bin/../lib/jruby-complete-1.6.8.jar:/opt/hbase/bin/../lib/jsch-0.1.42.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr305-1.3.9.jar:/opt/hbase/bin/../lib/junit-4.12.jar:/opt/hbase/bin/../lib/leveldbjni-all-1.8.jar:/opt/hbase/bin/../lib/libthrift-0.9.3.jar:/opt/hbase/bin/../lib/log4j-1.2.17.jar:/opt/hbase/bin/../lib/metrics-core-2.2.0.jar:/opt/hbase/bin/../lib/nekohtml-1.9.12.jar:/opt/hbase/bin/../lib/netty-all-4.0.23.Final.jar:/opt/hbase/bin/../lib/paranamer-2.3.jar:/opt/hbase/bin/../lib/protobuf-java-2.5.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/servlet-api-2.5.jar:/opt/hbase/bin/../lib/slf4j-api-1.7.7.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.7.5.jar:/opt/hbase/bin/../lib/snappy-java-1.0.4.1.jar:/opt/hbase/bin/../lib/spymemcached-2.11.6.jar:/opt/hbase/bin/../lib/xalan-2.7.0.jar:/opt/hbase/bin/../lib/xml-apis-1.3.03.jar:/opt/hbase/bin/../lib/xml-apis-ext-1.3.04.jar:/opt/hbase/bin/../lib/xmlenc-0.52.jar:/opt/hbase/bin/../lib/xom-1.2.5.jar:/opt/hbase/bin/../lib/xz-1.0.jar:/opt/hbase/bin/../lib/zookeeper-3.4.6.jar: +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.compiler= +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:os.name=Linux +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:os.arch=amd64 +hbase_1 | hbase stderr | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:os.version=5.3.0-62-generic +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:user.name=root +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:user.home=/root +hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:user.dir=/opt +hbase_1 | hbase stderr | 2020-07-22 09:39:22,002 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=master:412290x0, quorum=localhost:2181, baseZNode=/hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:22,020 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) +hbase_1 | hbase stderr | 2020-07-22 09:39:22,020 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34218 +hbase_1 | hbase stderr | 2020-07-22 09:39:22,021 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session +hbase_1 | hbase stderr | 2020-07-22 09:39:22,022 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34218 +hbase_1 | hbase stderr | 2020-07-22 09:39:22,030 INFO [SyncThread:0] persistence.FileTxnLog: Creating new log file: log.2100c +hbase_1 | hbase stderr | 2020-07-22 09:39:22,041 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0000 with negotiated timeout 10000 for client /127.0.0.1:34218 +hbase_1 | hbase stderr | 2020-07-22 09:39:22,042 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0000, negotiated timeout = 10000 +hbase_1 | hbase stderr | 2020-07-22 09:39:22,046 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x1 zxid:0x2100d txntype:-1 reqpath:n/a Error Path:/hbase Error:KeeperErrorCode = NodeExists for /hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:22,086 INFO [RpcServer.responder] ipc.RpcServer: RpcServer.responder: starting +hbase_1 | hbase stderr | 2020-07-22 09:39:22,090 INFO [RpcServer.listener,port=41229] ipc.RpcServer: RpcServer.listener,port=41229: starting +hbase_1 | hbase stderr | 2020-07-22 09:39:22,328 INFO [main] mortbay.log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog +hbase_1 | hbase stderr | 2020-07-22 09:39:22,333 INFO [main] http.HttpRequestLog: Http request log for http.requests.master is not defined +hbase_1 | hbase stderr | 2020-07-22 09:39:22,347 INFO [main] http.HttpServer: Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) +hbase_1 | hbase stderr | 2020-07-22 09:39:22,347 INFO [main] http.HttpServer: Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) +hbase_1 | hbase stderr | 2020-07-22 09:39:22,350 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master +hbase_1 | hbase stderr | 2020-07-22 09:39:22,350 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static +hbase_1 | 2020-07-22 09:39:22,350 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs +hbase_1 | hbase stderr | 2020-07-22 09:39:22,377 INFO [main] http.HttpServer: Jetty bound to port 16010 +hbase_1 | 2020-07-22 09:39:22,377 INFO [main] mortbay.log: jetty-6.1.26 +hbase_1 | hbase stderr | 2020-07-22 09:39:22,843 INFO [main] mortbay.log: Started SelectChannelConnector@0.0.0.0:16010 +hbase_1 | hbase stderr | 2020-07-22 09:39:22,853 INFO [main] master.HMaster: hbase.rootdir=file:/data/hbase, hbase.cluster.distributed=false +hbase_1 | hbase stderr | 2020-07-22 09:39:22,877 INFO [main] master.HMaster: Adding backup master ZNode /hbase/backup-masters/hbase,41229,1595410761736 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,033 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x10 zxid:0x2100f txntype:-1 reqpath:n/a Error Path:/hbase/master Error:KeeperErrorCode = NodeExists for /hbase/master +hbase_1 | hbase stderr | 2020-07-22 09:39:23,053 INFO [hbase:41229.activeMasterManager] master.ActiveMasterManager: Another master is the active master, hbase,41223,1595410680501; waiting to become the next active master +hbase_1 | hbase stderr | 2020-07-22 09:39:23,085 INFO [main] regionserver.RSRpcServices: regionserver/hbase/172.18.0.2:0 server-side HConnection retries=350 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,086 INFO [main] ipc.SimpleRpcScheduler: Using deadline as user call queue, count=3 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,147 INFO [main] ipc.RpcServer: regionserver/hbase/172.18.0.2:0: started 10 reader(s) listening on port=41633 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,149 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=regionserver:41633 connecting to ZooKeeper ensemble=localhost:2181 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,151 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=regionserver:416330x0, quorum=localhost:2181, baseZNode=/hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:23,158 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) +hbase_1 | hbase stderr | 2020-07-22 09:39:23,158 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34220 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,207 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session +hbase_1 | hbase stderr | 2020-07-22 09:39:23,208 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34220 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,219 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0001 with negotiated timeout 10000 for client /127.0.0.1:34220 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,219 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0001, negotiated timeout = 10000 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,227 INFO [RpcServer.responder] ipc.RpcServer: RpcServer.responder: starting +hbase_1 | hbase stderr | 2020-07-22 09:39:23,234 INFO [RpcServer.listener,port=41633] ipc.RpcServer: RpcServer.listener,port=41633: starting +hbase_1 | hbase stderr | 2020-07-22 09:39:23,298 INFO [main] http.HttpRequestLog: Http request log for http.requests.regionserver is not defined +hbase_1 | hbase stderr | 2020-07-22 09:39:23,299 INFO [main] http.HttpServer: Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) +hbase_1 | hbase stderr | 2020-07-22 09:39:23,299 INFO [main] http.HttpServer: Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) +elasticsearch | [2020-07-22T09:39:24,622][INFO ][o.e.d.DiscoveryModule ] [5q9Ine5] using discovery type [zen] and host providers [settings] +elasticsearch | [2020-07-22T09:39:25,037][INFO ][o.e.n.Node ] [5q9Ine5] initialized +elasticsearch | [2020-07-22T09:39:25,037][INFO ][o.e.n.Node ] [5q9Ine5] starting ... +elasticsearch | [2020-07-22T09:39:25,175][INFO ][o.e.t.TransportService ] [5q9Ine5] publish_address {172.18.0.3:9300}, bound_addresses {0.0.0.0:9300} +elasticsearch | [2020-07-22T09:39:25,196][INFO ][o.e.b.BootstrapChecks ] [5q9Ine5] bound or publishing to a non-loopback address, enforcing bootstrap checks +elasticsearch | [2020-07-22T09:39:28,230][INFO ][o.e.c.s.MasterService ] [5q9Ine5] zen-disco-elected-as-master ([0] nodes joined), reason: new_master {5q9Ine5}{5q9Ine5JToGzVTBOdMThhw}{_w0q-7V6SiWl02g_-m4GQA}{172.18.0.3}{172.18.0.3:9300} +elasticsearch | [2020-07-22T09:39:28,233][INFO ][o.e.c.s.ClusterApplierService] [5q9Ine5] new_master {5q9Ine5}{5q9Ine5JToGzVTBOdMThhw}{_w0q-7V6SiWl02g_-m4GQA}{172.18.0.3}{172.18.0.3:9300}, reason: apply cluster state (from master [master {5q9Ine5}{5q9Ine5JToGzVTBOdMThhw}{_w0q-7V6SiWl02g_-m4GQA}{172.18.0.3}{172.18.0.3:9300} committed version [1] source [zen-disco-elected-as-master ([0] nodes joined)]]) +elasticsearch | [2020-07-22T09:39:28,243][INFO ][o.e.h.n.Netty4HttpServerTransport] [5q9Ine5] publish_address {172.18.0.3:9200}, bound_addresses {0.0.0.0:9200} +elasticsearch | [2020-07-22T09:39:28,243][INFO ][o.e.n.Node ] [5q9Ine5] started +elasticsearch | [2020-07-22T09:39:28,386][INFO ][o.e.g.GatewayService ] [5q9Ine5] recovered [7] indices into cluster_state +elasticsearch | [2020-07-22T09:39:28,659][INFO ][o.e.c.r.a.AllocationService] [5q9Ine5] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[consoles_history][0], [table-meta][0], [consoles_v2][0], [consoles][0]] ...]). +hbase_1 | hbase stderr | 2020-07-22 09:39:23,300 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver +hbase_1 | hbase stderr | 2020-07-22 09:39:23,300 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs +hbase_1 | 2020-07-22 09:39:23,300 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static +hbase_1 | hbase stderr | 2020-07-22 09:39:23,303 INFO [main] http.HttpServer: Jetty bound to port 36787 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,303 INFO [main] mortbay.log: jetty-6.1.26 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,427 INFO [main] mortbay.log: Started SelectChannelConnector@0.0.0.0:36787 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,519 INFO [M:0;hbase:41229] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x33489efd connecting to ZooKeeper ensemble=localhost:2181 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,519 INFO [M:0;hbase:41229] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=hconnection-0x33489efd0x0, quorum=localhost:2181, baseZNode=/hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:23,523 INFO [M:0;hbase:41229-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) +hbase_1 | hbase stderr | 2020-07-22 09:39:23,523 INFO [M:0;hbase:41229-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session +hbase_1 | hbase stderr | 2020-07-22 09:39:23,526 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34222 +hbase_1 | 2020-07-22 09:39:23,528 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34222 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,530 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0002 with negotiated timeout 10000 for client /127.0.0.1:34222 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,530 INFO [M:0;hbase:41229-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0002, negotiated timeout = 10000 +hbase_1 | hbase stderr | 2020-07-22 09:39:23,563 INFO [M:0;hbase:41229] regionserver.HRegionServer: ClusterId : 0cbe9efc-c509-45a5-97a4-499957bb7b66 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770005, timeout of 10000ms exceeded +hbase_1 | hbase stderr | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770000, timeout of 10000ms exceeded +hbase_1 | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770004, timeout of 10000ms exceeded +hbase_1 | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770002, timeout of 10000ms exceeded +hbase_1 | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770003, timeout of 10000ms exceeded +hbase_1 | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770001, timeout of 10000ms exceeded +hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770005 +hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770000 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770004 +hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770002 +hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770003 +hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770001 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,010 INFO [hbase:41229.activeMasterManager] master.ActiveMasterManager: Deleting ZNode for /hbase/backup-masters/hbase,41229,1595410761736 from backup master directory +hbase_1 | hbase stderr | 2020-07-22 09:39:32,013 WARN [hbase:41229.activeMasterManager] hbase.ZNodeClearer: Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) +hbase_1 | 2020-07-22 09:39:32,013 INFO [hbase:41229.activeMasterManager] master.ActiveMasterManager: Registered Active Master=hbase,41229,1595410761736 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,056 INFO [hbase:41229.activeMasterManager] coordination.SplitLogManagerCoordination: Found 0 orphan tasks and 0 rescan nodes +hbase_1 | hbase stderr | 2020-07-22 09:39:32,060 INFO [RS:0;hbase:41633] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x561d3720 connecting to ZooKeeper ensemble=localhost:2181 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,060 INFO [RS:0;hbase:41633] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=hconnection-0x561d37200x0, quorum=localhost:2181, baseZNode=/hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:32,065 INFO [RS:0;hbase:41633-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) +hbase_1 | hbase stderr | 2020-07-22 09:39:32,066 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34328 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,066 INFO [RS:0;hbase:41633-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session +hbase_1 | hbase stderr | 2020-07-22 09:39:32,067 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34328 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,068 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0003 with negotiated timeout 10000 for client /127.0.0.1:34328 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,068 INFO [RS:0;hbase:41633-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0003, negotiated timeout = 10000 +hbase_1 | 2020-07-22 09:39:32,069 INFO [hbase:41229.activeMasterManager] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x445dd583 connecting to ZooKeeper ensemble=localhost:2181 +hbase_1 | 2020-07-22 09:39:32,069 INFO [hbase:41229.activeMasterManager] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=hconnection-0x445dd5830x0, quorum=localhost:2181, baseZNode=/hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:32,070 INFO [RS:0;hbase:41633] regionserver.HRegionServer: ClusterId : 0cbe9efc-c509-45a5-97a4-499957bb7b66 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,075 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0001 type:create cxid:0x9 zxid:0x2101c txntype:-1 reqpath:n/a Error Path:/hbase/flush-table-proc/acquired Error:KeeperErrorCode = NodeExists for /hbase/flush-table-proc/acquired +hbase_1 | hbase stderr | 2020-07-22 09:39:32,079 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0001 type:create cxid:0xc zxid:0x2101d txntype:-1 reqpath:n/a Error Path:/hbase/online-snapshot/acquired Error:KeeperErrorCode = NodeExists for /hbase/online-snapshot/acquired +hbase_1 | hbase stderr | 2020-07-22 09:39:32,085 INFO [RS:0;hbase:41633] regionserver.MemStoreFlusher: globalMemStoreLimit=3.1 G, globalMemStoreLimitLowMark=2.9 G, maxHeap=7.7 G +hbase_1 | hbase stderr | 2020-07-22 09:39:32,086 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) +hbase_1 | hbase stderr | 2020-07-22 09:39:32,087 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34330 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,087 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session +hbase_1 | hbase stderr | 2020-07-22 09:39:32,087 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34330 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,089 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0004 with negotiated timeout 10000 for client /127.0.0.1:34330 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,089 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0004, negotiated timeout = 10000 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,090 INFO [RS:0;hbase:41633] regionserver.HRegionServer: CompactionChecker runs every 10sec +hbase_1 | hbase stderr | 2020-07-22 09:39:32,098 INFO [hbase:41229.activeMasterManager] balancer.StochasticLoadBalancer: loading config +hbase_1 | hbase stderr | 2020-07-22 09:39:32,106 INFO [RS:0;hbase:41633] regionserver.RegionServerCoprocessorHost: System coprocessor loading is enabled +hbase_1 | 2020-07-22 09:39:32,106 INFO [RS:0;hbase:41633] regionserver.RegionServerCoprocessorHost: Table coprocessor loading is enabled +hbase_1 | hbase stderr | 2020-07-22 09:39:32,107 INFO [RS:0;hbase:41633] regionserver.HRegionServer: reportForDuty to master=hbase,41229,1595410761736 with port=41633, startcode=1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,187 INFO [hbase:41229.activeMasterManager] master.HMaster: Server active/primary master=hbase,41229,1595410761736, sessionid=0x17375e441fa0000, setting cluster-up flag (Was=true) +hbase_1 | hbase stderr | 2020-07-22 09:39:32,196 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x2f zxid:0x21020 txntype:-1 reqpath:n/a Error Path:/hbase/flush-table-proc/acquired Error:KeeperErrorCode = NodeExists for /hbase/flush-table-proc/acquired +hbase_1 | hbase stderr | 2020-07-22 09:39:32,198 INFO [hbase:41229.activeMasterManager] procedure.ZKProcedureUtil: Clearing all procedure znodes: /hbase/flush-table-proc/acquired /hbase/flush-table-proc/reached /hbase/flush-table-proc/abort +hbase_1 | hbase stderr | 2020-07-22 09:39:32,201 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x35 zxid:0x21021 txntype:-1 reqpath:n/a Error Path:/hbase/online-snapshot/acquired Error:KeeperErrorCode = NodeExists for /hbase/online-snapshot/acquired +hbase_1 | hbase stderr | 2020-07-22 09:39:32,202 INFO [hbase:41229.activeMasterManager] procedure.ZKProcedureUtil: Clearing all procedure znodes: /hbase/online-snapshot/acquired /hbase/online-snapshot/reached /hbase/online-snapshot/abort +hbase_1 | hbase stderr | 2020-07-22 09:39:32,222 INFO [hbase:41229.activeMasterManager] master.MasterCoprocessorHost: System coprocessor loading is enabled +hbase_1 | hbase stderr | 2020-07-22 09:39:32,234 INFO [hbase:41229.activeMasterManager] procedure2.ProcedureExecutor: Starting procedure executor threads=9 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,235 INFO [hbase:41229.activeMasterManager] wal.WALProcedureStore: Starting WAL Procedure Store lease recovery +hbase_1 | hbase stderr | 2020-07-22 09:39:32,237 WARN [hbase:41229.activeMasterManager] wal.WALProcedureStore: Remove uninitialized log: DeprecatedRawLocalFileStatus{path=file:/data/hbase/MasterProcWALs/state-00000000000000000219.log; isDirectory=false; length=0; replication=1; blocksize=33554432; modification_time=1595410713000; access_time=0; owner=; group=; permission=rw-rw-rw-; isSymlink=false} +foxtrot_server | done +foxtrot_server | Executing Init Command: java -jar server.jar initialize /config/docker.yml +foxtrot_server | ERROR StatusLogger Log4j2 could not find a logging implementation. Please add log4j-core to the classpath. Using SimpleLogger to log to the console... +foxtrot_server | INFO [2020-07-22 09:39:34,394] [main] [ElasticsearchConnection]: Starting ElasticSearch Client +foxtrot_server | INFO [2020-07-22 09:39:34,657] [main] [ElasticsearchConnection]: Started ElasticSearch Client +foxtrot_server | INFO [2020-07-22 09:39:34,804] [main] [InitializerCommand]: # data nodes: 1, Setting replica count to: 0 +foxtrot_server | INFO [2020-07-22 09:39:34,823] [main] [InitializerCommand]: Index consoles already exists. Nothing to do. +foxtrot_server | INFO [2020-07-22 09:39:34,830] [main] [InitializerCommand]: Index consoles_v2 already exists. Nothing to do. +foxtrot_server | INFO [2020-07-22 09:39:34,834] [main] [InitializerCommand]: Index table-meta already exists. Nothing to do. +foxtrot_server | INFO [2020-07-22 09:39:34,838] [main] [InitializerCommand]: Index consoles_history already exists. Nothing to do. +foxtrot_server | INFO [2020-07-22 09:39:34,842] [main] [InitializerCommand]: Index fql-store already exists. Nothing to do. +foxtrot_server | INFO [2020-07-22 09:39:34,846] [main] [InitializerCommand]: Index user-meta already exists. Nothing to do. +foxtrot_server | INFO [2020-07-22 09:39:34,850] [main] [InitializerCommand]: Index tokens already exists. Nothing to do. +foxtrot_server | INFO [2020-07-22 09:39:34,850] [main] [InitializerCommand]: Creating mapping +elasticsearch | [2020-07-22T09:39:34,960][INFO ][o.e.c.m.MetaDataIndexTemplateService] [5q9Ine5] adding template [template_foxtrot_mappings] for index patterns [foxtrot-*] +foxtrot_server | INFO [2020-07-22 09:39:34,966] [main] [InitializerCommand]: Created mapping: true +foxtrot_server | INFO [2020-07-22 09:39:34,966] [main] [ElasticsearchConnection]: Stopping ElasticSearch client +foxtrot_server | INFO [2020-07-22 09:39:34,968] [main] [InitializerCommand]: Creating hbase table +foxtrot_server | WARN [2020-07-22 09:39:35,221] [main] [NativeCodeLoader]: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable +hbase_1 | hbase stderr | 2020-07-22 09:39:32,246 INFO [hbase:41229.activeMasterManager] wal.WALProcedureStore: Lease acquired for flushLogId: 220 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,260 INFO [hbase:41229.activeMasterManager] zookeeper.RecoverableZooKeeper: Process identifier=replicationLogCleaner connecting to ZooKeeper ensemble=localhost:2181 +hbase_1 | 2020-07-22 09:39:32,260 INFO [hbase:41229.activeMasterManager] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=replicationLogCleaner0x0, quorum=localhost:2181, baseZNode=/hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:32,263 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) +hbase_1 | hbase stderr | 2020-07-22 09:39:32,263 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34332 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,264 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session +hbase_1 | hbase stderr | 2020-07-22 09:39:32,265 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34332 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,271 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0005 with negotiated timeout 10000 for client /127.0.0.1:34332 +hbase_1 | 2020-07-22 09:39:32,271 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0005, negotiated timeout = 10000 +hbase_1 | hbase stderr | 2020-07-22 09:39:32,276 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0005 type:create cxid:0x1 zxid:0x21023 txntype:-1 reqpath:n/a Error Path:/hbase/replication/rs Error:KeeperErrorCode = NodeExists for /hbase/replication/rs +hbase_1 | hbase stderr | 2020-07-22 09:39:32,284 INFO [hbase:41229.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 0, slept for 0 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms. +hbase_1 | hbase stderr | 2020-07-22 09:39:32,288 WARN [RS:0;hbase:41633] regionserver.HRegionServer: reportForDuty failed; sleeping and then retrying. +hbase_1 | hbase stderr | 2020-07-22 09:39:33,787 INFO [hbase:41229.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 0, slept for 1503 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms. +hbase_1 | hbase stderr | 2020-07-22 09:39:35,289 INFO [RS:0;hbase:41633] regionserver.HRegionServer: reportForDuty to master=hbase,41229,1595410761736 with port=41633, startcode=1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:35,290 INFO [hbase:41229.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 0, slept for 3006 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms. +hbase_1 | hbase stderr | 2020-07-22 09:39:35,302 INFO [B.defaultRpcServer.handler=5,queue=2,port=41229] master.ServerManager: Registering server=hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:35,312 WARN [RS:0;hbase:41633] hbase.ZNodeClearer: Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) +hbase_1 | hbase stderr | 2020-07-22 09:39:35,315 INFO [RS:0;hbase:41633] hfile.CacheConfig: Allocating LruBlockCache size=3.10 GB, blockSize=64 KB +hbase_1 | hbase stderr | 2020-07-22 09:39:35,324 INFO [RS:0;hbase:41633] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=0, currentSize=3412528, freeSize=3323299792, maxSize=3326712320, heapSize=3412528, minSize=3160376576, minFactor=0.95, multiSize=1580188288, multiFactor=0.5, singleSize=790094144, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false +hbase_1 | hbase stderr | 2020-07-22 09:39:35,340 INFO [hbase:41229.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 1, slept for 3056 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms. +hbase_1 | hbase stderr | 2020-07-22 09:39:35,372 INFO [RS:0;hbase:41633] wal.WALFactory: Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.DefaultWALProvider +hbase_1 | hbase stderr | 2020-07-22 09:39:35,385 INFO [RS:0;hbase:41633] wal.FSHLog: WAL configuration: blocksize=32 MB, rollsize=30.40 MB, prefix=hbase%2C41633%2C1595410763148.default, suffix=, logDir=file:/data/hbase/WALs/hbase,41633,1595410763148, archiveDir=file:/data/hbase/oldWALs +hbase_1 | hbase stderr | 2020-07-22 09:39:35,423 INFO [RS:0;hbase:41633] wal.FSHLog: Slow sync cost: 0 ms, current pipeline: [] +hbase_1 | hbase stderr | 2020-07-22 09:39:35,424 INFO [RS:0;hbase:41633] wal.FSHLog: New WAL /data/hbase/WALs/hbase,41633,1595410763148/hbase%2C41633%2C1595410763148.default.1595410775385 +hbase_1 | hbase stderr | 2020-07-22 09:39:35,450 INFO [RS:0;hbase:41633] regionserver.MetricsRegionServerWrapperImpl: Computing regionserver metrics every 5000 milliseconds +hbase_1 | hbase stderr | 2020-07-22 09:39:35,458 INFO [RS:0;hbase:41633] regionserver.ReplicationSourceManager: Current list of replicators: [hbase,37695,1595404637854, hbase,41633,1595410763148, hbase,43049,1595410682202] other RSs: [hbase,41633,1595410763148] +hbase_1 | hbase stderr | 2020-07-22 09:39:35,498 INFO [SplitLogWorker-hbase:41633] regionserver.SplitLogWorker: SplitLogWorker hbase,41633,1595410763148 starting +hbase_1 | 2020-07-22 09:39:35,500 INFO [RS:0;hbase:41633] regionserver.HeapMemoryManager: Starting HeapMemoryTuner chore. +hbase_1 | hbase stderr | 2020-07-22 09:39:35,502 INFO [RS:0;hbase:41633] regionserver.HRegionServer: Serving as hbase,41633,1595410763148, RpcServer on hbase/172.18.0.2:41633, sessionid=0x17375e441fa0001 +hbase_1 | hbase stderr | 2020-07-22 09:39:35,514 INFO [RS:0;hbase:41633] quotas.RegionServerQuotaManager: Quota support disabled +hbase_1 | hbase stderr | 2020-07-22 09:39:35,534 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53314 +hbase_1 | hbase stderr | 2020-07-22 09:39:35,537 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53314 +hbase_1 | hbase stderr | 2020-07-22 09:39:35,539 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0006 with negotiated timeout 40000 for client /172.18.0.4:53314 +hbase_1 | hbase stderr | 2020-07-22 09:39:36,845 INFO [hbase:41229.activeMasterManager] master.ServerManager: Finished waiting for region servers count to settle; checked in 1, slept for 4561 ms, expecting minimum of 1, maximum of 2147483647, master is running +hbase_1 | hbase stderr | 2020-07-22 09:39:36,846 INFO [hbase:41229.activeMasterManager] master.MasterFileSystem: Log folder file:/data/hbase/WALs/hbase,43049,1595410682202 doesn't belong to a known region server, splitting +hbase_1 | hbase stderr | 2020-07-22 09:39:36,847 INFO [hbase:41229.activeMasterManager] master.MasterFileSystem: Log folder file:/data/hbase/WALs/hbase,41633,1595410763148 belongs to an existing region server +hbase_1 | hbase stderr | 2020-07-22 09:39:36,850 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: dead splitlog workers [hbase,43049,1595410682202] +hbase_1 | hbase stderr | 2020-07-22 09:39:36,851 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: Started splitting 1 logs in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] for [hbase,43049,1595410682202] +hbase_1 | hbase stderr | 2020-07-22 09:39:36,861 INFO [SplitLogWorker-hbase:41633] coordination.ZkSplitLogWorkerCoordination: worker hbase,41633,1595410763148 acquired task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta +hbase_1 | hbase stderr | 2020-07-22 09:39:36,861 INFO [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta acquired by hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:36,885 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] wal.WALSplitter: Splitting wal: file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202..meta.1595410688276.meta, length=1016 +hbase_1 | 2020-07-22 09:39:36,886 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] wal.WALSplitter: DistributedLogReplay = false +hbase_1 | hbase stderr | 2020-07-22 09:39:36,939 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] wal.WALSplitter: 3 split writers finished; closing... +hbase_1 | hbase stderr | 2020-07-22 09:39:36,943 INFO [split-log-closeStream-1] wal.WALSplitter: Rename file:/data/hbase/data/hbase/meta/1588230740/recovered.edits/0000000000000000677.temp to file:/data/hbase/data/hbase/meta/1588230740/recovered.edits/0000000000000000678 +hbase_1 | hbase stderr | 2020-07-22 09:39:36,944 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] wal.WALSplitter: Processed 2 edits across 1 regions; edits skipped=1; log file=file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202..meta.1595410688276.meta, length=1016, corrupted=false, progress failed=false +hbase_1 | hbase stderr | 2020-07-22 09:39:36,947 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] coordination.ZkSplitLogWorkerCoordination: successfully transitioned task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta to final state DONE hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:36,947 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] handler.WALSplitterHandler: worker hbase,41633,1595410763148 done with task org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination$ZkSplitTaskDetails@bb64aec in 84ms +hbase_1 | 2020-07-22 09:39:36,947 INFO [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta entered state: DONE hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:36,948 INFO [main-EventThread] wal.WALSplitter: Archived processed log file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202..meta.1595410688276.meta to file:/data/hbase/oldWALs/hbase%2C43049%2C1595410682202..meta.1595410688276.meta +hbase_1 | hbase stderr | 2020-07-22 09:39:36,949 INFO [main-EventThread] coordination.SplitLogManagerCoordination: Done splitting /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta +hbase_1 | hbase stderr | 2020-07-22 09:39:36,950 WARN [hbase:41229.activeMasterManager] master.SplitLogManager: Returning success without actually splitting and deleting all the log files in path file:/data/hbase/WALs/hbase,43049,1595410682202-splitting: [DeprecatedRawLocalFileStatus{path=file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976; isDirectory=false; length=669; replication=1; blocksize=33554432; modification_time=1595410689000; access_time=0; owner=; group=; permission=rw-rw-rw-; isSymlink=false}] +hbase_1 | java.io.IOException: Directory /data/hbase/WALs/hbase,43049,1595410682202-splitting is not empty +hbase_1 | at org.apache.hadoop.fs.RawLocalFileSystem.delete(RawLocalFileSystem.java:360) +hbase_1 | at org.apache.hadoop.fs.ChecksumFileSystem.delete(ChecksumFileSystem.java:540) +hbase_1 | at org.apache.hadoop.hbase.master.SplitLogManager.splitLogDistributed(SplitLogManager.java:296) +hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitLog(MasterFileSystem.java:398) +hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitMetaLog(MasterFileSystem.java:313) +hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitMetaLog(MasterFileSystem.java:304) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster.splitMetaLogBeforeAssignment(HMaster.java:989) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:695) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster.access$500(HMaster.java:184) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster$1.run(HMaster.java:1729) +hbase_1 | at java.lang.Thread.run(Thread.java:745) +hbase_1 | hbase stderr | 2020-07-22 09:39:36,951 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: finished splitting (more than or equal to) 1016 bytes in 1 log files in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] in 100ms +hbase_1 | hbase stderr | 2020-07-22 09:39:36,964 INFO [hbase:41229.activeMasterManager] zookeeper.MetaTableLocator: Failed verification of hbase:meta,,1 at address=hbase,43049,1595410682202, exception=Connection refused +hbase_1 | hbase stderr | 2020-07-22 09:39:36,965 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: dead splitlog workers [hbase,43049,1595410682202] +hbase_1 | hbase stderr | 2020-07-22 09:39:36,965 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: file:/data/hbase/WALs/hbase,43049,1595410682202-splitting is empty dir, no logs to split +hbase_1 | hbase stderr | 2020-07-22 09:39:36,965 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: Started splitting 0 logs in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] for [hbase,43049,1595410682202] +hbase_1 | hbase stderr | 2020-07-22 09:39:36,965 WARN [hbase:41229.activeMasterManager] master.SplitLogManager: Returning success without actually splitting and deleting all the log files in path file:/data/hbase/WALs/hbase,43049,1595410682202-splitting: [DeprecatedRawLocalFileStatus{path=file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976; isDirectory=false; length=669; replication=1; blocksize=33554432; modification_time=1595410689000; access_time=0; owner=; group=; permission=rw-rw-rw-; isSymlink=false}] +hbase_1 | java.io.IOException: Directory /data/hbase/WALs/hbase,43049,1595410682202-splitting is not empty +hbase_1 | at org.apache.hadoop.fs.RawLocalFileSystem.delete(RawLocalFileSystem.java:360) +hbase_1 | at org.apache.hadoop.fs.ChecksumFileSystem.delete(ChecksumFileSystem.java:540) +hbase_1 | at org.apache.hadoop.hbase.master.SplitLogManager.splitLogDistributed(SplitLogManager.java:296) +hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitLog(MasterFileSystem.java:398) +hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitMetaLog(MasterFileSystem.java:313) +hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitMetaLog(MasterFileSystem.java:304) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster.splitMetaLogBeforeAssignment(HMaster.java:989) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster.assignMeta(HMaster.java:917) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:728) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster.access$500(HMaster.java:184) +hbase_1 | at org.apache.hadoop.hbase.master.HMaster$1.run(HMaster.java:1729) +hbase_1 | at java.lang.Thread.run(Thread.java:745) +hbase_1 | hbase stderr | 2020-07-22 09:39:36,966 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: finished splitting (more than or equal to) 0 bytes in 0 log files in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] in 1ms +hbase_1 | 2020-07-22 09:39:36,966 INFO [hbase:41229.activeMasterManager] zookeeper.MetaTableLocator: Deleting hbase:meta region location in ZooKeeper +hbase_1 | hbase stderr | 2020-07-22 09:39:36,969 INFO [hbase:41229.activeMasterManager] master.AssignmentManager: Setting node as OFFLINED in ZooKeeper for region {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} +hbase_1 | hbase stderr | 2020-07-22 09:39:36,975 INFO [hbase:41229.activeMasterManager] master.AssignmentManager: Assigning hbase:meta,,1.1588230740 to hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:36,975 INFO [hbase:41229.activeMasterManager] master.RegionStates: Transition {1588230740 state=OFFLINE, ts=1595410776969, server=null} to {1588230740 state=PENDING_OPEN, ts=1595410776975, server=hbase,41633,1595410763148} +hbase_1 | hbase stderr | 2020-07-22 09:39:36,985 INFO [PriorityRpcServer.handler=0,queue=0,port=41633] regionserver.RSRpcServices: Open hbase:meta,,1.1588230740 +hbase_1 | hbase stderr | 2020-07-22 09:39:36,990 INFO [hbase:41229.activeMasterManager] master.ServerManager: AssignmentManager hasn't finished failover cleanup; waiting +hbase_1 | hbase stderr | 2020-07-22 09:39:36,994 INFO [RS_OPEN_META-hbase:41633-0] wal.WALFactory: Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.DefaultWALProvider +hbase_1 | hbase stderr | 2020-07-22 09:39:36,995 INFO [RS_OPEN_META-hbase:41633-0] wal.FSHLog: WAL configuration: blocksize=32 MB, rollsize=30.40 MB, prefix=hbase%2C41633%2C1595410763148..meta, suffix=.meta, logDir=file:/data/hbase/WALs/hbase,41633,1595410763148, archiveDir=file:/data/hbase/oldWALs +hbase_1 | hbase stderr | 2020-07-22 09:39:36,996 INFO [AM.ZK.Worker-pool2-t1] master.RegionStates: Transition {1588230740 state=PENDING_OPEN, ts=1595410776975, server=hbase,41633,1595410763148} to {1588230740 state=OPENING, ts=1595410776996, server=hbase,41633,1595410763148} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,006 INFO [RS_OPEN_META-hbase:41633-0] wal.FSHLog: Slow sync cost: 0 ms, current pipeline: [] +hbase_1 | hbase stderr | 2020-07-22 09:39:37,006 INFO [RS_OPEN_META-hbase:41633-0] wal.FSHLog: New WAL /data/hbase/WALs/hbase,41633,1595410763148/hbase%2C41633%2C1595410763148..meta.1595410776995.meta +hbase_1 | hbase stderr | 2020-07-22 09:39:37,041 INFO [RS_OPEN_META-hbase:41633-0] regionserver.RegionCoprocessorHost: Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. +hbase_1 | hbase stderr | 2020-07-22 09:39:37,084 INFO [StoreOpener-1588230740-1] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=0, currentSize=3412528, freeSize=3323299792, maxSize=3326712320, heapSize=3412528, minSize=3160376576, minFactor=0.95, multiSize=1580188288, multiFactor=0.5, singleSize=790094144, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false +hbase_1 | hbase stderr | 2020-07-22 09:39:37,090 INFO [StoreOpener-1588230740-1] compactions.CompactionConfiguration: size [134217728, 9223372036854775807, 9223372036854775807); files [3, 10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,136 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HRegion: Replaying edits from file:/data/hbase/data/hbase/meta/1588230740/recovered.edits/0000000000000000678 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,141 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HRegion: Flushing 1/1 column families, memstore=1.38 KB; WAL is null, using passed sequenceid=678 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,189 INFO [RS_OPEN_META-hbase:41633-0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=678, memsize=1.4 K, hasBloomFilter=false, into tmp file file:/data/hbase/data/hbase/meta/1588230740/.tmp/ec2b6cf702f1488f8d734d1e8a2a5d6a +hbase_1 | hbase stderr | 2020-07-22 09:39:37,210 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HStore: Added file:/data/hbase/data/hbase/meta/1588230740/info/ec2b6cf702f1488f8d734d1e8a2a5d6a, entries=6, sequenceid=678, filesize=5.3 K +hbase_1 | hbase stderr | 2020-07-22 09:39:37,211 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HRegion: Finished memstore flush of ~1.38 KB/1416, currentsize=0 B/0 for region hbase:meta,,1.1588230740 in 71ms, sequenceid=678, compaction requested=false; wal=null +hbase_1 | hbase stderr | 2020-07-22 09:39:37,223 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HRegion: Onlined 1588230740; next sequenceid=679 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,234 INFO [PostOpenDeployTasks:1588230740] regionserver.HRegionServer: Post open deploy tasks for hbase:meta,,1.1588230740 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,235 INFO [PostOpenDeployTasks:1588230740] zookeeper.MetaTableLocator: Setting hbase:meta region location in ZooKeeper as hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,236 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0001 type:setData cxid:0x39 zxid:0x2102e txntype:-1 reqpath:n/a Error Path:/hbase/meta-region-server Error:KeeperErrorCode = NoNode for /hbase/meta-region-server +hbase_1 | hbase stderr | 2020-07-22 09:39:37,241 INFO [AM.ZK.Worker-pool2-t2] master.RegionStates: Transition {1588230740 state=OPENING, ts=1595410776996, server=hbase,41633,1595410763148} to {1588230740 state=OPEN, ts=1595410777241, server=hbase,41633,1595410763148} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,242 INFO [AM.ZK.Worker-pool2-t2] coordination.ZkOpenRegionCoordination: Handling OPENED of 1588230740 from hbase,41229,1595410761736; deleting unassigned node +hbase_1 | hbase stderr | 2020-07-22 09:39:37,249 INFO [hbase:41229.activeMasterManager] master.HMaster: hbase:meta with replicaId 0 assigned=1, rit=false, location=hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,309 INFO [hbase:41229.activeMasterManager] hbase.MetaMigrationConvertingToPB: META already up-to date with PB serialization +hbase_1 | hbase stderr | 2020-07-22 09:39:37,322 INFO [hbase:41229.activeMasterManager] master.RegionStates: Transition {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OPEN, ts=1595410777322, server=hbase,43049,1595410682202} to {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OFFLINE, ts=1595410777322, server=hbase,43049,1595410682202} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,322 INFO [hbase:41229.activeMasterManager] master.RegionStates: Transition {d0d5370755b75937494d142505e5a2a4 state=OPEN, ts=1595410777322, server=hbase,43049,1595410682202} to {d0d5370755b75937494d142505e5a2a4 state=OFFLINE, ts=1595410777322, server=hbase,43049,1595410682202} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,323 INFO [hbase:41229.activeMasterManager] master.AssignmentManager: Found regions out on cluster or in RIT; presuming failover +hbase_1 | hbase stderr | 2020-07-22 09:39:37,442 INFO [hbase:41229.activeMasterManager] master.AssignmentManager: Joined the cluster in 133ms, failover=true +hbase_1 | hbase stderr | 2020-07-22 09:39:37,443 INFO [ProcedureExecutor-0] procedure.ServerCrashProcedure: Start processing crashed hbase,43049,1595410682202 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,650 INFO [ProcedureExecutor-2] master.SplitLogManager: dead splitlog workers [hbase,43049,1595410682202] +hbase_1 | hbase stderr | 2020-07-22 09:39:37,651 INFO [ProcedureExecutor-2] master.SplitLogManager: Started splitting 1 logs in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] for [hbase,43049,1595410682202] +hbase_1 | hbase stderr | 2020-07-22 09:39:37,655 INFO [SplitLogWorker-hbase:41633] coordination.ZkSplitLogWorkerCoordination: worker hbase,41633,1595410763148 acquired task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,655 INFO [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 acquired by hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,669 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] wal.WALSplitter: Splitting wal: file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976, length=669 +hbase_1 | 2020-07-22 09:39:37,669 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] wal.WALSplitter: DistributedLogReplay = false +hbase_1 | hbase stderr | 2020-07-22 09:39:37,686 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] wal.WALSplitter: 3 split writers finished; closing... +hbase_1 | hbase stderr | 2020-07-22 09:39:37,686 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] wal.WALSplitter: Processed 0 edits across 0 regions; edits skipped=2; log file=file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976, length=669, corrupted=false, progress failed=false +hbase_1 | hbase stderr | 2020-07-22 09:39:37,687 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] coordination.ZkSplitLogWorkerCoordination: successfully transitioned task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 to final state DONE hbase,41633,1595410763148 +foxtrot_server | INFO [2020-07-22 09:39:37,696] [main] [HBaseUtil]: Table foxtrot already exists. Nothing to do. +foxtrot_server | INFO [2020-07-22 09:39:37,709] [main] [InitializerCommand]: Initialization complete... +foxtrot_server | Starting foxtrot with command line: java -Dfile.encoding=utf-8 -XX:+UseG1GC -Xms1g -Xmx1g -jar server.jar server /config/docker.yml +hbase_1 | hbase stderr | 2020-07-22 09:39:37,687 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] handler.WALSplitterHandler: worker hbase,41633,1595410763148 done with task org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination$ZkSplitTaskDetails@7dfa1b92 in 32ms +hbase_1 | hbase stderr | 2020-07-22 09:39:37,687 INFO [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 entered state: DONE hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,688 INFO [main-EventThread] wal.WALSplitter: Archived processed log file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976 to file:/data/hbase/oldWALs/hbase%2C43049%2C1595410682202.default.1595410683976 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,688 INFO [main-EventThread] coordination.SplitLogManagerCoordination: Done splitting /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,689 INFO [ProcedureExecutor-2] master.SplitLogManager: finished splitting (more than or equal to) 669 bytes in 1 log files in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] in 38ms +hbase_1 | hbase stderr | 2020-07-22 09:39:37,697 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e441fa0006 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,699 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxn: Closed socket connection for client /172.18.0.4:53314 which had sessionid 0x17375e441fa0006 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,791 INFO [ProcedureExecutor-2] master.AssignmentManager: Assigning 2 region(s) to hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,798 INFO [ProcedureExecutor-2] master.RegionStates: Transition {d0d5370755b75937494d142505e5a2a4 state=OFFLINE, ts=1595410777793, server=hbase,43049,1595410682202} to {d0d5370755b75937494d142505e5a2a4 state=PENDING_OPEN, ts=1595410777798, server=hbase,41633,1595410763148} +hbase_1 | 2020-07-22 09:39:37,798 INFO [ProcedureExecutor-2] master.RegionStates: Transition {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OFFLINE, ts=1595410777793, server=hbase,43049,1595410682202} to {d7c0a1ab1664e7d809c8b3dcacccddf7 state=PENDING_OPEN, ts=1595410777798, server=hbase,41633,1595410763148} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,801 INFO [PriorityRpcServer.handler=2,queue=0,port=41633] regionserver.RSRpcServices: Open hbase:namespace,,1586108027618.d0d5370755b75937494d142505e5a2a4. +hbase_1 | hbase stderr | 2020-07-22 09:39:37,802 INFO [PriorityRpcServer.handler=2,queue=0,port=41633] regionserver.RSRpcServices: Open foxtrot,,1586108035639.d7c0a1ab1664e7d809c8b3dcacccddf7. +hbase_1 | hbase stderr | 2020-07-22 09:39:37,807 INFO [AM.ZK.Worker-pool2-t5] master.RegionStates: Transition {d0d5370755b75937494d142505e5a2a4 state=PENDING_OPEN, ts=1595410777798, server=hbase,41633,1595410763148} to {d0d5370755b75937494d142505e5a2a4 state=OPENING, ts=1595410777807, server=hbase,41633,1595410763148} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,811 INFO [AM.ZK.Worker-pool2-t6] master.RegionStates: Transition {d7c0a1ab1664e7d809c8b3dcacccddf7 state=PENDING_OPEN, ts=1595410777798, server=hbase,41633,1595410763148} to {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OPENING, ts=1595410777811, server=hbase,41633,1595410763148} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,814 INFO [RS_OPEN_REGION-hbase:41633-1] compress.CodecPool: Got brand-new compressor [.gz] +hbase_1 | hbase stderr | 2020-07-22 09:39:37,826 INFO [StoreOpener-d0d5370755b75937494d142505e5a2a4-1] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=2, currentSize=3421240, freeSize=3323291080, maxSize=3326712320, heapSize=3421240, minSize=3160376576, minFactor=0.95, multiSize=1580188288, multiFactor=0.5, singleSize=790094144, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false +hbase_1 | hbase stderr | 2020-07-22 09:39:37,826 INFO [StoreOpener-d0d5370755b75937494d142505e5a2a4-1] compactions.CompactionConfiguration: size [134217728, 9223372036854775807, 9223372036854775807); files [3, 10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,838 INFO [StoreOpener-d7c0a1ab1664e7d809c8b3dcacccddf7-1] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=2, currentSize=3421240, freeSize=3323291080, maxSize=3326712320, heapSize=3421240, minSize=3160376576, minFactor=0.95, multiSize=1580188288, multiFactor=0.5, singleSize=790094144, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false +hbase_1 | hbase stderr | 2020-07-22 09:39:37,839 INFO [StoreOpener-d7c0a1ab1664e7d809c8b3dcacccddf7-1] compactions.CompactionConfiguration: size [134217728, 9223372036854775807, 9223372036854775807); files [3, 10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,847 INFO [StoreFileOpenerThread-d-1] compress.CodecPool: Got brand-new decompressor [.gz] +hbase_1 | hbase stderr | 2020-07-22 09:39:37,850 INFO [StoreFileOpenerThread-d-1] compress.CodecPool: Got brand-new decompressor [.gz] +hbase_1 | hbase stderr | 2020-07-22 09:39:37,850 INFO [StoreFileOpenerThread-d-1] compress.CodecPool: Got brand-new decompressor [.gz] +hbase_1 | hbase stderr | 2020-07-22 09:39:37,850 INFO [StoreFileOpenerThread-d-1] compress.CodecPool: Got brand-new decompressor [.gz] +hbase_1 | hbase stderr | 2020-07-22 09:39:37,871 INFO [RS_OPEN_REGION-hbase:41633-1] regionserver.HRegion: Onlined d7c0a1ab1664e7d809c8b3dcacccddf7; next sequenceid=10236 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,873 INFO [RS_OPEN_REGION-hbase:41633-0] regionserver.HRegion: Onlined d0d5370755b75937494d142505e5a2a4; next sequenceid=195 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,882 INFO [PostOpenDeployTasks:d7c0a1ab1664e7d809c8b3dcacccddf7] regionserver.HRegionServer: Post open deploy tasks for foxtrot,,1586108035639.d7c0a1ab1664e7d809c8b3dcacccddf7. +hbase_1 | hbase stderr | 2020-07-22 09:39:37,892 INFO [PostOpenDeployTasks:d0d5370755b75937494d142505e5a2a4] regionserver.HRegionServer: Post open deploy tasks for hbase:namespace,,1586108027618.d0d5370755b75937494d142505e5a2a4. +hbase_1 | hbase stderr | 2020-07-22 09:39:37,961 INFO [PostOpenDeployTasks:d7c0a1ab1664e7d809c8b3dcacccddf7] hbase.MetaTableAccessor: Updated row foxtrot,,1586108035639.d7c0a1ab1664e7d809c8b3dcacccddf7. with server=hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,961 INFO [PostOpenDeployTasks:d0d5370755b75937494d142505e5a2a4] hbase.MetaTableAccessor: Updated row hbase:namespace,,1586108027618.d0d5370755b75937494d142505e5a2a4. with server=hbase,41633,1595410763148 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,975 INFO [AM.ZK.Worker-pool2-t8] master.RegionStates: Transition {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OPENING, ts=1595410777811, server=hbase,41633,1595410763148} to {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OPEN, ts=1595410777975, server=hbase,41633,1595410763148} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,976 INFO [AM.ZK.Worker-pool2-t7] master.RegionStates: Transition {d0d5370755b75937494d142505e5a2a4 state=OPENING, ts=1595410777807, server=hbase,41633,1595410763148} to {d0d5370755b75937494d142505e5a2a4 state=OPEN, ts=1595410777976, server=hbase,41633,1595410763148} +hbase_1 | hbase stderr | 2020-07-22 09:39:37,979 INFO [AM.ZK.Worker-pool2-t10] master.RegionStates: Offlined d7c0a1ab1664e7d809c8b3dcacccddf7 from hbase,43049,1595410682202 +hbase_1 | hbase stderr | 2020-07-22 09:39:37,980 INFO [AM.ZK.Worker-pool2-t11] master.RegionStates: Offlined d0d5370755b75937494d142505e5a2a4 from hbase,43049,1595410682202 +hbase_1 | hbase stderr | 2020-07-22 09:39:38,082 INFO [ProcedureExecutor-2] procedure.ServerCrashProcedure: Finished processing of crashed hbase,43049,1595410682202 +foxtrot_server | ERROR StatusLogger Log4j2 could not find a logging implementation. Please add log4j-core to the classpath. Using SimpleLogger to log to the console... +foxtrot_server | INFO [2020-07-22 09:39:39,444] [main] [DefaultServerFactory]: Registering jersey handler with root path prefix: / +foxtrot_server | INFO [2020-07-22 09:39:39,447] [main] [DefaultServerFactory]: Registering admin handler with root path prefix: / +foxtrot_server | INFO [2020-07-22 09:39:39,447] [main] [AssetsBundle]: Registering AssetBundle with name: console for path /* +foxtrot_server | INFO [2020-07-22 09:39:39,474] [main] [AssetsBundle]: Registering AssetBundle with name: swagger-assets for path /foxtrot/swagger-static/* +foxtrot_server | INFO [2020-07-22 09:39:39,474] [main] [AssetsBundle]: Registering AssetBundle with name: swagger-oauth2-connect for path /foxtrot/oauth2-redirect.html/* +foxtrot_server | INFO [2020-07-22 09:39:43,919] [main] [Reflections]: Reflections took 4424 ms to scan 1 urls, producing 79344 keys and 149875 values +foxtrot_server | WARN [2020-07-22 09:39:44,666] [main] [ExecutorServiceBuilder]: Parameter 'maximumPoolSize' is conflicting with unbounded work queues +foxtrot_server | INFO [2020-07-22 09:39:44,690] [main] [ManagedInstaller]: managed = +foxtrot_server | +foxtrot_server | (com.flipkart.foxtrot.core.datastore.impl.hbase.HbaseTableConnection) +foxtrot_server | (com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection) +foxtrot_server | (com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection) +foxtrot_server | (com.flipkart.foxtrot.core.table.impl.DistributedTableMetadataManager) +foxtrot_server | (com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader) +foxtrot_server | (com.flipkart.foxtrot.core.common.DataDeletionManager) +foxtrot_server | (com.flipkart.foxtrot.server.cluster.ClusterManager) +foxtrot_server | (com.flipkart.foxtrot.core.cardinality.CardinalityCalculationManager) +foxtrot_server | (com.flipkart.foxtrot.core.internalevents.EventBusInitializer) +foxtrot_server | (com.flipkart.foxtrot.core.jobs.optimization.EsIndexOptimizationManager) +foxtrot_server | (com.flipkart.foxtrot.server.jobs.consolehistory.ConsoleHistoryManager) +foxtrot_server | (com.flipkart.foxtrot.server.auth.sessionstore.DistributedSessionDataStore) +foxtrot_server | (com.flipkart.foxtrot.server.jobs.sessioncleanup.ExpiredSessionsCleaner) +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:44,691] [main] [HealthCheckInstaller]: health checks = +foxtrot_server | +foxtrot_server | hazelcastHealthcheck (com.flipkart.foxtrot.server.healthcheck.HazelcastHealthCheck) +foxtrot_server | hbaseHealthcheck (com.flipkart.foxtrot.server.healthcheck.HBaseHealthCheck) +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:44,692] [main] [WebFilterInstaller]: filters = +foxtrot_server | +foxtrot_server | /* (com.flipkart.foxtrot.server.auth.filter.UserAuthenticationFilter) .userauthentication +foxtrot_server | +foxtrot_server | WARN [2020-07-22 09:39:44,725] [main] [NativeCodeLoader]: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable +foxtrot_server | INFO [2020-07-22 09:39:45,426] [main] [ServerFactory]: Starting foxtrot +foxtrot_server | _______ ______ ___ ___ .___________..______ ______ .___________. +foxtrot_server | | ____| / __ \ \ \ / / | || _ \ / __ \ | | +foxtrot_server | | |__ | | | | \ V / `---| |----`| |_) | | | | | `---| |----` +foxtrot_server | | __| | | | | > < | | | / | | | | | | +foxtrot_server | | | | `--' | / . \ | | | |\ \----.| `--' | | | +foxtrot_server | |__| \______/ /__/ \__\ |__| | _| `._____| \______/ |__| +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:45,532] [main] [SetUIDListener]: Opened application@7b2c82c{HTTP/1.1,[http/1.1]}{0.0.0.0:17000} +foxtrot_server | INFO [2020-07-22 09:39:45,532] [main] [SetUIDListener]: Opened admin@30fa6986{HTTP/1.1,[http/1.1]}{0.0.0.0:17001} +foxtrot_server | INFO [2020-07-22 09:39:45,535] [main] [Server]: jetty-9.4.z-SNAPSHOT; built: 2019-04-29T20:42:08.989Z; git: e1bc35120a6617ee3df052294e433f3a25ce7097; jvm 1.8.0_252-8u252-b09-1~16.04-b09 +foxtrot_server | INFO [2020-07-22 09:39:45,549] [main] [HbaseTableConnection]: Starting HBase Connection +foxtrot_server | INFO [2020-07-22 09:39:45,641] [main] [HbaseTableConnection]: Started HBase Connection +foxtrot_server | INFO [2020-07-22 09:39:45,641] [main] [ElasticsearchConnection]: Starting ElasticSearch Client +foxtrot_server | INFO [2020-07-22 09:39:46,241] [main] [ElasticsearchConnection]: Started ElasticSearch Client +foxtrot_server | INFO [2020-07-22 09:39:46,242] [main] [HazelcastConnection]: Starting Hazelcast Instance +foxtrot_server | WARN [2020-07-22 09:39:46,342] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] You configured your member address as host name. Please be aware of that your dns can be spoofed. Make sure that your dns configurations are correct. +foxtrot_server | INFO [2020-07-22 09:39:46,342] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] Resolving domain name 'fac2f46c5b7f' to address(es): [172.18.0.4] +foxtrot_server | INFO [2020-07-22 09:39:46,342] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] Interfaces is disabled, trying to pick one address from TCP-IP config addresses: [fac2f46c5b7f/172.18.0.4] +foxtrot_server | INFO [2020-07-22 09:39:46,342] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] Prefer IPv4 stack is true, prefer IPv6 addresses is false +foxtrot_server | INFO [2020-07-22 09:39:46,345] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] Picked [fac2f46c5b7f]:5701, using socket ServerSocket[addr=/0.0.0.0,localport=5701], bind any local is true +foxtrot_server | INFO [2020-07-22 09:39:46,375] [main] [system]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Hazelcast 4.0.2 (20200702 - 2de3027) starting at [fac2f46c5b7f]:5701 +foxtrot_server | INFO [2020-07-22 09:39:46,375] [main] [system]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Copyright (c) 2008-2020, Hazelcast, Inc. All Rights Reserved. +foxtrot_server | INFO [2020-07-22 09:39:46,694] [main] [BackpressureRegulator]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Backpressure is disabled +foxtrot_server | INFO [2020-07-22 09:39:47,116] [main] [Node]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Creating TcpIpJoiner +foxtrot_server | WARN [2020-07-22 09:39:47,118] [main] [CPSubsystem]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] CP Subsystem is not enabled. CP data structures will operate in UNSAFE mode! Please note that UNSAFE mode will not provide strong consistency guarantees. +foxtrot_server | INFO [2020-07-22 09:39:47,423] [main] [OperationExecutorImpl]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Starting 8 partition threads and 5 generic threads (1 dedicated for priority tasks) +foxtrot_server | INFO [2020-07-22 09:39:47,432] [main] [Diagnostics]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Diagnostics disabled. To enable add -Dhazelcast.diagnostics.enabled=true to the JVM arguments. +foxtrot_server | INFO [2020-07-22 09:39:47,446] [main] [LifecycleService]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] [fac2f46c5b7f]:5701 is STARTING +foxtrot_server | INFO [2020-07-22 09:39:47,480] [main] [ClusterService]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] +foxtrot_server | +foxtrot_server | Members {size:1, ver:1} [ +foxtrot_server | Member [fac2f46c5b7f]:5701 - 259b100b-6b86-4551-a0cf-775a4676c26d this +foxtrot_server | ] +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:47,495] [main] [LifecycleService]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] [fac2f46c5b7f]:5701 is STARTED +foxtrot_server | INFO [2020-07-22 09:39:47,495] [main] [HazelcastConnection]: Started Hazelcast Instance +foxtrot_server | INFO [2020-07-22 09:39:47,526] [main] [PartitionStateManager]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Initializing cluster partition table arrangement... +foxtrot_server | INFO [2020-07-22 09:39:47,613] [hz.foxtrot-fac2f46c5b7f-1595410784519.cached.thread-3] [TableMapStore]: Load all keys called +foxtrot_server | INFO [2020-07-22 09:39:47,927] [hz.foxtrot-fac2f46c5b7f-1595410784519.cached.thread-3] [TableMapStore]: Loaded value count: 0 +foxtrot_server | INFO [2020-07-22 09:39:48,446] [main] [Reflections]: Reflections took 130 ms to scan 1 urls, producing 86 keys and 237 values +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.TrendAction +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.FilterAction +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.MultiTimeQueryAction +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.DistinctAction +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.MultiQueryAction +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.HistogramAction +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.StatsAction +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.CountAction +foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.StatsTrendAction +foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.GroupAction +foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [DataDeletionManager]: Starting Deletion Manager +foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [DataDeletionManager]: Scheduling data deletion Job +foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [DataDeletionManager]: Scheduled data deletion Job +foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [DataDeletionManager]: Started Deletion Manager +foxtrot_server | INFO [2020-07-22 09:39:48,449] [main] [BaseJobManager]: Starting CardinalityConfigJob Manager +foxtrot_server | INFO [2020-07-22 09:39:48,449] [main] [BaseJobManager]: Scheduling CardinalityConfigJob Job +foxtrot_server | INFO [2020-07-22 09:39:48,452] [main] [BaseJobManager]: Scheduled CardinalityConfigJob Job +foxtrot_server | INFO [2020-07-22 09:39:48,454] [main] [BaseJobManager]: Starting ESIndexOptimizer Manager +foxtrot_server | INFO [2020-07-22 09:39:48,454] [main] [BaseJobManager]: Scheduling ESIndexOptimizer Job +foxtrot_server | INFO [2020-07-22 09:39:48,454] [main] [BaseJobManager]: Scheduled ESIndexOptimizer Job +foxtrot_server | INFO [2020-07-22 09:39:48,455] [main] [BaseJobManager]: Starting ConsoleHistory Manager +foxtrot_server | INFO [2020-07-22 09:39:48,455] [main] [BaseJobManager]: Scheduling ConsoleHistory Job +foxtrot_server | INFO [2020-07-22 09:39:48,455] [main] [BaseJobManager]: Scheduled ConsoleHistory Job +foxtrot_server | INFO [2020-07-22 09:39:48,456] [main] [BaseJobManager]: Starting ExpiredSessionCleaner Manager +foxtrot_server | INFO [2020-07-22 09:39:48,456] [main] [BaseJobManager]: Scheduling ExpiredSessionCleaner Job +foxtrot_server | INFO [2020-07-22 09:39:48,456] [main] [BaseJobManager]: Scheduled ExpiredSessionCleaner Job +foxtrot_server | INFO [2020-07-22 09:39:48,704] [main] [JerseyProviderInstaller]: providers = +foxtrot_server | +foxtrot_server | Exception mappers +foxtrot_server | JWTAuthenticationFailure (com.flipkart.foxtrot.server.providers.exception.AuthenticationExceptionHandler) +foxtrot_server | FoxtrotException (com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper) +foxtrot_server | +foxtrot_server | Dynamic features +foxtrot_server | (com.flipkart.foxtrot.server.auth.filter.JwtAuthDynamicFeature) +foxtrot_server | +foxtrot_server | Message body writers +foxtrot_server | FlatRepresentation (com.flipkart.foxtrot.server.providers.FlatResponseCsvProvider) +foxtrot_server | Map (com.flipkart.foxtrot.server.providers.FlatResponseErrorTextProvider) +foxtrot_server | FlatRepresentation (com.flipkart.foxtrot.server.providers.FlatResponseTextProvider) +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:49,090] [main] [DropwizardResourceConfig]: The following paths were found for the configured resources: +foxtrot_server | +foxtrot_server | GET /foxtrot/google/callback (com.flipkart.foxtrot.server.resources.GoogleAuth) +foxtrot_server | GET /foxtrot/google/login (com.flipkart.foxtrot.server.resources.GoogleAuth) +foxtrot_server | GET /foxtrot/swagger (io.federecio.dropwizard.swagger.SwaggerResource) +foxtrot_server | GET /foxtrot/swagger.{type:json|yaml} (io.swagger.jaxrs.listing.ApiListingResource) +foxtrot_server | POST /foxtrot/v1/analytics (com.flipkart.foxtrot.server.resources.AnalyticsResource) +foxtrot_server | POST /foxtrot/v1/analytics/async (com.flipkart.foxtrot.server.resources.AnalyticsResource) +foxtrot_server | POST /foxtrot/v1/analytics/download (com.flipkart.foxtrot.server.resources.AnalyticsResource) +foxtrot_server | POST /foxtrot/v1/analytics/validate (com.flipkart.foxtrot.server.resources.AnalyticsResource) +foxtrot_server | POST /foxtrot/v1/async (com.flipkart.foxtrot.server.resources.AsyncResource) +foxtrot_server | GET /foxtrot/v1/async/{action}/{id} (com.flipkart.foxtrot.server.resources.AsyncResource) +foxtrot_server | GET /foxtrot/v1/auth/tokens/{tokenId} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | DELETE /foxtrot/v1/auth/tokens/{userId} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | POST /foxtrot/v1/auth/tokens/{userId} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | POST /foxtrot/v1/auth/users (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | DELETE /foxtrot/v1/auth/users/{userId} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | GET /foxtrot/v1/auth/users/{userId} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | PUT /foxtrot/v1/auth/users/{userId}/roles/grant/{role} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | PUT /foxtrot/v1/auth/users/{userId}/roles/revoke/{role} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | PUT /foxtrot/v1/auth/users/{userId}/tables/access/grant/{table} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | PUT /foxtrot/v1/auth/users/{userId}/tables/access/revoke/{table} (com.flipkart.foxtrot.server.resources.Auth) +foxtrot_server | POST /foxtrot/v1/cache/update/cardinality (com.flipkart.foxtrot.server.resources.CacheUpdateResource) +foxtrot_server | GET /foxtrot/v1/cluster/hazelcast/members (com.flipkart.foxtrot.server.resources.ClusterInfoResource) +foxtrot_server | GET /foxtrot/v1/cluster/members (com.flipkart.foxtrot.server.resources.ClusterInfoResource) +foxtrot_server | GET /foxtrot/v1/clusterhealth (com.flipkart.foxtrot.server.resources.ClusterHealthResource) +foxtrot_server | GET /foxtrot/v1/clusterhealth/indicesstats (com.flipkart.foxtrot.server.resources.ClusterHealthResource) +foxtrot_server | GET /foxtrot/v1/clusterhealth/nodestats (com.flipkart.foxtrot.server.resources.ClusterHealthResource) +foxtrot_server | GET /foxtrot/v1/consoles (com.flipkart.foxtrot.server.resources.ConsoleResource) +foxtrot_server | POST /foxtrot/v1/consoles (com.flipkart.foxtrot.server.resources.ConsoleResource) +foxtrot_server | GET /foxtrot/v1/consoles/{id} (com.flipkart.foxtrot.server.resources.ConsoleResource) +foxtrot_server | DELETE /foxtrot/v1/consoles/{id}/delete (com.flipkart.foxtrot.server.resources.ConsoleResource) +foxtrot_server | GET /foxtrot/v1/document/{table} (com.flipkart.foxtrot.server.resources.DocumentResource) +foxtrot_server | POST /foxtrot/v1/document/{table} (com.flipkart.foxtrot.server.resources.DocumentResource) +foxtrot_server | POST /foxtrot/v1/document/{table}/bulk (com.flipkart.foxtrot.server.resources.DocumentResource) +foxtrot_server | GET /foxtrot/v1/document/{table}/{id} (com.flipkart.foxtrot.server.resources.DocumentResource) +foxtrot_server | POST /foxtrot/v1/fql (com.flipkart.foxtrot.server.resources.FqlResource) +foxtrot_server | GET /foxtrot/v1/fql/download (com.flipkart.foxtrot.server.resources.FqlResource) +foxtrot_server | POST /foxtrot/v1/fql/get (com.flipkart.foxtrot.server.resources.FqlResource) +foxtrot_server | POST /foxtrot/v1/fql/save (com.flipkart.foxtrot.server.resources.FqlResource) +foxtrot_server | GET /foxtrot/v1/hbase/regions/{table}/{threshSizeInGB}/list (com.flipkart.foxtrot.server.resources.HbaseRegionsMergeResource) +foxtrot_server | GET /foxtrot/v1/hbase/regions/{table}/{threshSizeInGB}/merge/{number} (com.flipkart.foxtrot.server.resources.HbaseRegionsMergeResource) +foxtrot_server | GET /foxtrot/v1/tables (com.flipkart.foxtrot.server.resources.TableManagerResource) +foxtrot_server | POST /foxtrot/v1/tables (com.flipkart.foxtrot.server.resources.TableManagerResource) +foxtrot_server | GET /foxtrot/v1/tables/fields (com.flipkart.foxtrot.server.resources.TableFieldMappingResource) +foxtrot_server | GET /foxtrot/v1/tables/{name} (com.flipkart.foxtrot.server.resources.TableManagerResource) +foxtrot_server | PUT /foxtrot/v1/tables/{name} (com.flipkart.foxtrot.server.resources.TableManagerResource) +foxtrot_server | DELETE /foxtrot/v1/tables/{name}/delete (com.flipkart.foxtrot.server.resources.TableManagerResource) +foxtrot_server | GET /foxtrot/v1/tables/{name}/fields (com.flipkart.foxtrot.server.resources.TableFieldMappingResource) +foxtrot_server | POST /foxtrot/v1/tables/{name}/fields/update (com.flipkart.foxtrot.server.resources.TableFieldMappingResource) +foxtrot_server | GET /foxtrot/v1/util/config (com.flipkart.foxtrot.server.resources.UtilResource) +foxtrot_server | GET /foxtrot/v2/consoles (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) +foxtrot_server | POST /foxtrot/v2/consoles (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) +foxtrot_server | GET /foxtrot/v2/consoles/{id} (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) +foxtrot_server | DELETE /foxtrot/v2/consoles/{id}/delete (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) +foxtrot_server | DELETE /foxtrot/v2/consoles/{id}/old/delete (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) +foxtrot_server | GET /foxtrot/v2/consoles/{id}/old/get (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) +foxtrot_server | GET /foxtrot/v2/consoles/{id}/old/set/current (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) +foxtrot_server | GET /foxtrot/v2/consoles/{name}/old (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) +foxtrot_server | +foxtrot_server | WARN [2020-07-22 09:39:49,095] [main] [Errors]: The following warnings have been detected: WARNING: A resource, Resource{"/v1/escluster", 0 child resources, 0 resource methods, 0 sub-resource locator, 0 method handler classes, 0 method handler instances}, with path "/v1/escluster" is empty. It has no resource (or sub resource) methods neither sub resource locators defined. +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:49,096] [main] [ContextHandler]: Started i.d.j.MutableServletContextHandler@6db18ce7{Application context,/,null,AVAILABLE} +foxtrot_server | INFO [2020-07-22 09:39:49,099] [main] [AdminEnvironment]: tasks = +foxtrot_server | +foxtrot_server | POST /tasks/OorTask (io.dropwizard.oor.tasks.OorTask) +foxtrot_server | POST /tasks/log-level (io.dropwizard.servlets.tasks.LogConfigurationTask) +foxtrot_server | POST /tasks/gc (io.dropwizard.servlets.tasks.GarbageCollectionTask) +foxtrot_server | POST /tasks/BirTask (io.dropwizard.oor.tasks.BirTask) +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:49,103] [main] [ContextHandler]: Started i.d.j.MutableServletContextHandler@42c32195{Admin context,/,null,AVAILABLE} +foxtrot_server | INFO [2020-07-22 09:39:49,110] [main] [AbstractConnector]: Started application@7b2c82c{HTTP/1.1,[http/1.1]}{0.0.0.0:17000} +foxtrot_server | INFO [2020-07-22 09:39:49,118] [main] [AbstractConnector]: Started admin@30fa6986{HTTP/1.1,[http/1.1]}{0.0.0.0:17001} +foxtrot_server | INFO [2020-07-22 09:39:49,118] [main] [Server]: Started @11390ms +foxtrot_server | INFO [2020-07-22 09:39:49,127] [main] [DiagnosticReporter]: Startup stats = +foxtrot_server | +foxtrot_server | GUICEY started in 1.584 s +foxtrot_server | │ +foxtrot_server | ├── [20%] CLASSPATH scanned in 310.8 ms +foxtrot_server | │ ├── scanned 309 classes +foxtrot_server | │ └── recognized 38 classes (12% of scanned) +foxtrot_server | │ +foxtrot_server | ├── [0.32%] BUNDLES processed in 5.771 ms +foxtrot_server | │ └── 3 processed +foxtrot_server | │ +foxtrot_server | ├── [77%] INJECTOR created in 1.223 s +foxtrot_server | │ ├── installers prepared in 4.723 ms +foxtrot_server | │ │ +foxtrot_server | │ ├── extensions recognized in 9.870 ms +foxtrot_server | │ │ ├── using 12 installers +foxtrot_server | │ │ └── from 309 classes +foxtrot_server | │ │ +foxtrot_server | │ └── 38 extensions installed in 5.604 ms +foxtrot_server | │ +foxtrot_server | ├── [0.25%] HK2 bridged in 4.429 ms +foxtrot_server | │ ├── using 2 jersey installers +foxtrot_server | │ └── 22 jersey extensions installed in 3.666 ms +foxtrot_server | │ +foxtrot_server | └── [2.7%] remaining 42 ms +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:49,130] [main] [DiagnosticReporter]: Options = +foxtrot_server | +foxtrot_server | Guicey (r.v.dropwizard.guice.GuiceyOptions) +foxtrot_server | ScanPackages = [com.flipkart.foxtrot] *CUSTOM +foxtrot_server | SearchCommands = false +foxtrot_server | UseCoreInstallers = true +foxtrot_server | ConfigureFromDropwizardBundles = false +foxtrot_server | BindConfigurationInterfaces = false +foxtrot_server | BindConfigurationByPath = true +foxtrot_server | InjectorStage = PRODUCTION *CUSTOM +foxtrot_server | GuiceFilterRegistration = [REQUEST] +foxtrot_server | UseHkBridge = false +foxtrot_server | +foxtrot_server | +foxtrot_server | Installers (r.v.d.g.m.i.InstallersOptions) +foxtrot_server | JerseyExtensionsManagedByGuice = true +foxtrot_server | ForceSingletonForJerseyExtensions = true +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:49,135] [main] [DiagnosticReporter]: Configuration diagnostic info = +foxtrot_server | +foxtrot_server | BUNDLES = +foxtrot_server | WebInstallersBundle (r.v.d.g.m.installer) +foxtrot_server | DiagnosticBundle (r.v.d.g.m.c.debug) +foxtrot_server | CoreInstallersBundle (r.v.d.g.m.installer) +foxtrot_server | +foxtrot_server | +foxtrot_server | INSTALLERS and EXTENSIONS in processing order = +foxtrot_server | managed (r.v.d.g.m.i.feature.ManagedInstaller) +foxtrot_server | HbaseTableConnection (c.f.f.c.d.impl.hbase) *SCAN +foxtrot_server | ElasticsearchConnection (c.f.f.c.q.impl) *SCAN +foxtrot_server | HazelcastConnection (c.f.f.c.q.impl) *SCAN +foxtrot_server | DistributedTableMetadataManager (c.f.f.c.table.impl) *SCAN +foxtrot_server | AnalyticsLoader (c.f.f.c.q.a.spi) *SCAN +foxtrot_server | DataDeletionManager (c.f.f.core.common) *SCAN +foxtrot_server | ClusterManager (c.f.f.server.cluster) *SCAN +foxtrot_server | CardinalityCalculationManager (c.f.f.c.cardinality) *SCAN +foxtrot_server | EventBusInitializer (c.f.f.c.internalevents) *SCAN +foxtrot_server | EsIndexOptimizationManager (c.f.f.c.j.optimization) *SCAN +foxtrot_server | ConsoleHistoryManager (c.f.f.s.j.consolehistory) *SCAN +foxtrot_server | DistributedSessionDataStore (c.f.f.s.a.sessionstore) *SCAN +foxtrot_server | ExpiredSessionsCleaner (c.f.f.s.j.sessioncleanup) *SCAN +foxtrot_server | jerseyprovider (r.v.d.g.m.i.f.j.p.JerseyProviderInstaller) +foxtrot_server | JwtAuthDynamicFeature (c.f.f.s.auth.filter) *SCAN +foxtrot_server | FlatResponseCsvProvider (c.f.f.s.providers) *SCAN +foxtrot_server | FlatResponseErrorTextProvider (c.f.f.s.providers) *SCAN +foxtrot_server | AuthenticationExceptionHandler (c.f.f.s.p.exception) *SCAN +foxtrot_server | FoxtrotExceptionMapper (c.f.f.s.p.exception) *SCAN +foxtrot_server | FlatResponseTextProvider (c.f.f.s.providers) *SCAN +foxtrot_server | resource (r.v.d.g.m.i.f.j.ResourceInstaller) +foxtrot_server | UtilResource (c.f.f.s.resources) *SCAN +foxtrot_server | Auth (c.f.f.s.resources) *SCAN +foxtrot_server | DocumentResource (c.f.f.s.resources) *SCAN +foxtrot_server | CacheUpdateResource (c.f.f.s.resources) *SCAN +foxtrot_server | ClusterInfoResource (c.f.f.s.resources) *SCAN +foxtrot_server | AsyncResource (c.f.f.s.resources) *SCAN +foxtrot_server | HbaseRegionsMergeResource (c.f.f.s.resources) *SCAN +foxtrot_server | TableFieldMappingResource (c.f.f.s.resources) *SCAN +foxtrot_server | ESClusterResource (c.f.f.s.resources) *SCAN +foxtrot_server | ConsoleV2Resource (c.f.f.s.resources) *SCAN +foxtrot_server | GoogleAuth (c.f.f.s.resources) *SCAN +foxtrot_server | FqlResource (c.f.f.s.resources) *SCAN +foxtrot_server | TableManagerResource (c.f.f.s.resources) *SCAN +foxtrot_server | ConsoleResource (c.f.f.s.resources) *SCAN +foxtrot_server | ClusterHealthResource (c.f.f.s.resources) *SCAN +foxtrot_server | AnalyticsResource (c.f.f.s.resources) *SCAN +foxtrot_server | healthcheck (r.v.d.g.m.i.f.h.HealthCheckInstaller) +foxtrot_server | HazelcastHealthCheck (c.f.f.s.healthcheck) *SCAN +foxtrot_server | HBaseHealthCheck (c.f.f.s.healthcheck) *SCAN +foxtrot_server | webfilter (r.v.d.g.m.i.f.web.WebFilterInstaller) +foxtrot_server | UserAuthenticationFilter (c.f.f.s.auth.filter) *SCAN +foxtrot_server | +foxtrot_server | +foxtrot_server | GUICE MODULES = +foxtrot_server | FoxtrotModule (c.f.f.server.di) +foxtrot_server | DiagnosticModule (r.v.d.g.m.c.d.DiagnosticBundle) +foxtrot_server | GuiceBootstrapModule (r.v.d.guice.module) +foxtrot_server | +foxtrot_server | INFO [2020-07-22 09:39:49,156] [main] [DiagnosticReporter]: Configuration context tree = +foxtrot_server | +foxtrot_server | APPLICATION +foxtrot_server | ├── module FoxtrotModule (c.f.f.server.di) +foxtrot_server | ├── module GuiceBootstrapModule (r.v.d.guice.module) +foxtrot_server | │ +foxtrot_server | ├── WebInstallersBundle (r.v.d.g.m.installer) +foxtrot_server | │ └── installer WebFilterInstaller (r.v.d.g.m.i.f.web) +foxtrot_server | │ +foxtrot_server | ├── DiagnosticBundle (r.v.d.g.m.c.debug) +foxtrot_server | │ └── module DiagnosticModule (r.v.d.g.m.c.d.DiagnosticBundle) +foxtrot_server | │ +foxtrot_server | ├── CoreInstallersBundle (r.v.d.g.m.installer) +foxtrot_server | │ ├── installer ManagedInstaller (r.v.d.g.m.i.feature) +foxtrot_server | │ ├── installer JerseyProviderInstaller (r.v.d.g.m.i.f.j.provider) +foxtrot_server | │ ├── installer ResourceInstaller (r.v.d.g.m.i.f.jersey) +foxtrot_server | │ └── installer HealthCheckInstaller (r.v.d.g.m.i.f.health) +foxtrot_server | │ +foxtrot_server | └── CLASSPATH SCAN +foxtrot_server | ├── extension UserAuthenticationFilter (c.f.f.s.auth.filter) +foxtrot_server | ├── extension JwtAuthDynamicFeature (c.f.f.s.auth.filter) +foxtrot_server | ├── extension DistributedSessionDataStore (c.f.f.s.a.sessionstore) +foxtrot_server | ├── extension HazelcastHealthCheck (c.f.f.s.healthcheck) +foxtrot_server | ├── extension HBaseHealthCheck (c.f.f.s.healthcheck) +foxtrot_server | ├── extension UtilResource (c.f.f.s.resources) +foxtrot_server | ├── extension Auth (c.f.f.s.resources) +foxtrot_server | ├── extension DocumentResource (c.f.f.s.resources) +foxtrot_server | ├── extension CacheUpdateResource (c.f.f.s.resources) +foxtrot_server | ├── extension ClusterInfoResource (c.f.f.s.resources) +foxtrot_server | ├── extension AsyncResource (c.f.f.s.resources) +foxtrot_server | ├── extension HbaseRegionsMergeResource (c.f.f.s.resources) +foxtrot_server | ├── extension TableFieldMappingResource (c.f.f.s.resources) +foxtrot_server | ├── extension ESClusterResource (c.f.f.s.resources) +foxtrot_server | ├── extension ConsoleV2Resource (c.f.f.s.resources) +foxtrot_server | ├── extension GoogleAuth (c.f.f.s.resources) +foxtrot_server | ├── extension FqlResource (c.f.f.s.resources) +foxtrot_server | ├── extension TableManagerResource (c.f.f.s.resources) +foxtrot_server | ├── extension ConsoleResource (c.f.f.s.resources) +foxtrot_server | ├── extension ClusterHealthResource (c.f.f.s.resources) +foxtrot_server | ├── extension AnalyticsResource (c.f.f.s.resources) +foxtrot_server | ├── extension ConsoleHistoryManager (c.f.f.s.j.consolehistory) +foxtrot_server | ├── extension ExpiredSessionsCleaner (c.f.f.s.j.sessioncleanup) +foxtrot_server | ├── extension FlatResponseCsvProvider (c.f.f.s.providers) +foxtrot_server | ├── extension FlatResponseErrorTextProvider (c.f.f.s.providers) +foxtrot_server | ├── extension AuthenticationExceptionHandler (c.f.f.s.p.exception) +foxtrot_server | ├── extension FoxtrotExceptionMapper (c.f.f.s.p.exception) +foxtrot_server | ├── extension FlatResponseTextProvider (c.f.f.s.providers) +foxtrot_server | ├── extension ClusterManager (c.f.f.server.cluster) +foxtrot_server | ├── extension HbaseTableConnection (c.f.f.c.d.impl.hbase) +foxtrot_server | ├── extension CardinalityCalculationManager (c.f.f.c.cardinality) +foxtrot_server | ├── extension AnalyticsLoader (c.f.f.c.q.a.spi) +foxtrot_server | ├── extension ElasticsearchConnection (c.f.f.c.q.impl) +foxtrot_server | ├── extension HazelcastConnection (c.f.f.c.q.impl) +foxtrot_server | ├── extension EventBusInitializer (c.f.f.c.internalevents) +foxtrot_server | ├── extension EsIndexOptimizationManager (c.f.f.c.j.optimization) +foxtrot_server | ├── extension DistributedTableMetadataManager (c.f.f.c.table.impl) +foxtrot_server | └── extension DataDeletionManager (c.f.f.core.common) +foxtrot_server | +foxtrot_server | 172.18.0.1 - - [22/Jul/2020:09:39:49 +0000] "GET /foxtrot/v1/clusterhealth HTTP/1.1" 200 3223 "http://localhost:17000/cluster/index.htm" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" 114 +foxtrot_server | 172.18.0.1 - - [22/Jul/2020:09:39:49 +0000] "GET /foxtrot/v1/clusterhealth/nodestats HTTP/1.1" 200 6879 "http://localhost:17000/cluster/index.htm" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" 114 +hbase_1 | hbase stderr | 2020-07-22 09:39:38,102 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x4ef zxid:0x21040 txntype:-1 reqpath:n/a Error Path:/hbase/namespace/default Error:KeeperErrorCode = NodeExists for /hbase/namespace/default +hbase_1 | hbase stderr | 2020-07-22 09:39:38,112 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x4f2 zxid:0x21042 txntype:-1 reqpath:n/a Error Path:/hbase/namespace/hbase Error:KeeperErrorCode = NodeExists for /hbase/namespace/hbase +hbase_1 | hbase stderr | 2020-07-22 09:39:38,113 INFO [hbase:41229.activeMasterManager] master.HMaster: Master has completed initialization +hbase_1 | hbase stderr | 2020-07-22 09:39:38,115 INFO [hbase:41229.activeMasterManager] quotas.MasterQuotaManager: Quota support disabled +hbase_1 | hbase stderr | 2020-07-22 09:39:38,115 INFO [hbase:41229.activeMasterManager] zookeeper.ZooKeeperWatcher: not a secure deployment, proceeding +hbase_1 | hbase stderr | 2020-07-22 09:39:42,046 INFO [HBase-Metrics2-1] impl.MetricsSystemImpl: Stopping HBase metrics system... +hbase_1 | hbase stderr | 2020-07-22 09:39:42,049 INFO [HBase-Metrics2-1] impl.MetricsSystemImpl: HBase metrics system stopped. +hbase_1 | hbase stderr | 2020-07-22 09:39:42,549 INFO [HBase-Metrics2-1] impl.MetricsConfig: loaded properties from hadoop-metrics2-hbase.properties +hbase_1 | hbase stderr | 2020-07-22 09:39:42,550 INFO [HBase-Metrics2-1] impl.MetricsSystemImpl: Scheduled snapshot period at 10 second(s). +hbase_1 | 2020-07-22 09:39:42,550 INFO [HBase-Metrics2-1] impl.MetricsSystemImpl: HBase metrics system started +hbase_1 | hbase stderr | 2020-07-22 09:39:45,165 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53410 +hbase_1 | hbase stderr | 2020-07-22 09:39:45,172 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53410 +hbase_1 | hbase stderr | 2020-07-22 09:39:45,174 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0007 with negotiated timeout 40000 for client /172.18.0.4:53410 +hbase_1 | hbase stderr | 2020-07-22 09:39:45,607 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53412 +hbase_1 | hbase stderr | 2020-07-22 09:39:45,626 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53412 +hbase_1 | hbase stderr | 2020-07-22 09:39:45,628 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0008 with negotiated timeout 40000 for client /172.18.0.4:53412 +hbase_1 | hbase stderr | 2020-07-22 09:39:48,505 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53424 +hbase_1 | hbase stderr | 2020-07-22 09:39:48,509 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53424 +hbase_1 | hbase stderr | 2020-07-22 09:39:48,510 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0009 with negotiated timeout 40000 for client /172.18.0.4:53424 +hbase_1 | hbase stderr | 2020-07-22 09:39:48,650 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e441fa0009 +hbase_1 | hbase stderr | 2020-07-22 09:39:48,651 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxn: Closed socket connection for client /172.18.0.4:53424 which had sessionid 0x17375e441fa0009 +hbase_1 | hbase stderr | 2020-07-22 09:39:53,693 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53434 +hbase_1 | hbase stderr | 2020-07-22 09:39:53,694 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53434 +hbase_1 | hbase stderr | 2020-07-22 09:39:53,696 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa000a with negotiated timeout 40000 for client /172.18.0.4:53434 +hbase_1 | hbase stderr | 2020-07-22 09:39:53,700 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e441fa000a +foxtrot_server | 172.18.0.1 - - [22/Jul/2020:09:39:57 +0000] "GET /foxtrot/v1/clusterhealth HTTP/1.1" 200 3223 "http://localhost:17000/cluster/index.htm" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" 13 +elasticsearch | [2020-07-22T09:39:58,244][WARN ][o.e.c.r.a.DiskThresholdMonitor] [5q9Ine5] high disk watermark [90%] exceeded on [5q9Ine5JToGzVTBOdMThhw][5q9Ine5][/usr/share/elasticsearch/data/nodes/0] free: 2.4gb[8.5%], shards will be relocated away from this node +elasticsearch | [2020-07-22T09:39:58,245][INFO ][o.e.c.r.a.DiskThresholdMonitor] [5q9Ine5] rerouting shards: [high disk watermark exceeded on one or more nodes] +foxtrot_server | 172.18.0.1 - - [22/Jul/2020:09:39:59 +0000] "GET /foxtrot/v1/clusterhealth/nodestats HTTP/1.1" 200 6881 "http://localhost:17000/cluster/index.htm" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" 35 diff --git a/scripts/cluster.json b/scripts/cluster.json new file mode 100644 index 000000000..bb5d8c78a --- /dev/null +++ b/scripts/cluster.json @@ -0,0 +1,496 @@ +{ + "nodes" : { + "CfmMTq98QSynR6K7VRQtSA" : { + "roles" : [ + "master", + "data", + "ingest" + ], + "adaptive_selection" : { + "CfmMTq98QSynR6K7VRQtSA" : { + "rank" : "11.7", + "avg_service_time_ns" : 10247513, + "outgoing_searches" : 0, + "avg_response_time_ns" : 11653187, + "avg_queue_size" : 0 + } + }, + "discovery" : { + "cluster_state_queue" : { + "committed" : 0, + "total" : 0, + "pending" : 0 + }, + "published_cluster_states" : { + "full_states" : 0, + "incompatible_diffs" : 0, + "compatible_diffs" : 0 + } + }, + "ip" : "172.18.0.2:9300", + "indices" : { + "request_cache" : { + "memory_size_in_bytes" : 0, + "miss_count" : 0, + "hit_count" : 0, + "evictions" : 0 + }, + "get" : { + "exists_time_in_millis" : 9, + "current" : 0, + "total" : 1, + "exists_total" : 1, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "time_in_millis" : 9 + }, + "refresh" : { + "total" : 18, + "listeners" : 0, + "total_time_in_millis" : 0 + }, + "merges" : { + "total_stopped_time_in_millis" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_auto_throttle_in_bytes" : 188743680, + "current" : 0, + "total_throttled_time_in_millis" : 0, + "total_time_in_millis" : 0, + "total_size_in_bytes" : 0, + "current_docs" : 0, + "total_docs" : 0 + }, + "search" : { + "suggest_current" : 0, + "scroll_time_in_millis" : 452616, + "query_current" : 0, + "scroll_total" : 3, + "fetch_time_in_millis" : 21, + "query_time_in_millis" : 16, + "suggest_time_in_millis" : 0, + "scroll_current" : 0, + "query_total" : 4, + "fetch_current" : 0, + "fetch_total" : 4, + "open_contexts" : 0, + "suggest_total" : 0 + }, + "query_cache" : { + "hit_count" : 0, + "evictions" : 0, + "miss_count" : 0, + "memory_size_in_bytes" : 0, + "cache_count" : 0, + "total_count" : 0, + "cache_size" : 0 + }, + "fielddata" : { + "evictions" : 0, + "memory_size_in_bytes" : 0 + }, + "indexing" : { + "index_current" : 0, + "delete_total" : 0, + "throttle_time_in_millis" : 0, + "delete_time_in_millis" : 0, + "index_failed" : 0, + "is_throttled" : false, + "noop_update_total" : 0, + "index_total" : 0, + "delete_current" : 0, + "index_time_in_millis" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "flush" : { + "total_time_in_millis" : 0, + "total" : 0, + "periodic" : 0 + }, + "segments" : { + "fixed_bit_set_memory_in_bytes" : 0, + "index_writer_memory_in_bytes" : 0, + "points_memory_in_bytes" : 22, + "terms_memory_in_bytes" : 3053, + "norms_memory_in_bytes" : 256, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : {}, + "memory_in_bytes" : 4091, + "stored_fields_memory_in_bytes" : 624, + "doc_values_memory_in_bytes" : 136, + "count" : 2, + "version_map_memory_in_bytes" : 0, + "term_vectors_memory_in_bytes" : 0 + }, + "warmer" : { + "total" : 9, + "current" : 0, + "total_time_in_millis" : 0 + }, + "store" : { + "size_in_bytes" : 14538 + }, + "recovery" : { + "throttle_time_in_millis" : 0, + "current_as_target" : 0, + "current_as_source" : 0 + }, + "docs" : { + "count" : 2, + "deleted" : 0 + }, + "translog" : { + "earliest_last_modified_age" : 0, + "uncommitted_operations" : 0, + "size_in_bytes" : 1485, + "operations" : 0, + "uncommitted_size_in_bytes" : 1485 + } + }, + "name" : "CfmMTq9", + "jvm" : { + "threads" : { + "count" : 55, + "peak_count" : 57 + }, + "timestamp" : 1586197973965, + "classes" : { + "total_loaded_count" : 12363, + "current_loaded_count" : 12363, + "total_unloaded_count" : 0 + }, + "buffer_pools" : { + "mapped" : { + "count" : 2, + "used_in_bytes" : 10533, + "total_capacity_in_bytes" : 10533 + }, + "direct" : { + "total_capacity_in_bytes" : 50562117, + "used_in_bytes" : 50562118, + "count" : 26 + }, + "mapped - 'non-volatile memory'" : { + "count" : 0, + "total_capacity_in_bytes" : 0, + "used_in_bytes" : 0 + } + }, + "gc" : { + "collectors" : { + "old" : { + "collection_time_in_millis" : 0, + "collection_count" : 0 + }, + "young" : { + "collection_count" : 6, + "collection_time_in_millis" : 72 + } + } + }, + "mem" : { + "heap_used_percent" : 22, + "heap_committed_in_bytes" : 536870912, + "non_heap_used_in_bytes" : 82819536, + "pools" : { + "old" : { + "used_in_bytes" : 96844800, + "max_in_bytes" : 536870912, + "peak_used_in_bytes" : 96844800, + "peak_max_in_bytes" : 536870912 + }, + "survivor" : { + "used_in_bytes" : 12058624, + "max_in_bytes" : 0, + "peak_used_in_bytes" : 40894464, + "peak_max_in_bytes" : 0 + }, + "young" : { + "used_in_bytes" : 13631488, + "max_in_bytes" : 0, + "peak_max_in_bytes" : 0, + "peak_used_in_bytes" : 230686720 + } + }, + "heap_max_in_bytes" : 536870912, + "non_heap_committed_in_bytes" : 89219072, + "heap_used_in_bytes" : 122534912 + }, + "uptime_in_millis" : 270334 + }, + "http" : { + "current_open" : 1, + "total_opened" : 1 + }, + "timestamp" : 1586197973956, + "host" : "172.18.0.2", + "ingest" : { + "total" : { + "current" : 0, + "time_in_millis" : 0, + "failed" : 0, + "count" : 0 + }, + "pipelines" : {} + }, + "fs" : { + "least_usage_estimate" : { + "used_disk_percent" : 62.9089363866555, + "path" : "/usr/share/elasticsearch/data/nodes/0", + "total_in_bytes" : 30633005056, + "available_in_bytes" : 11362107392 + }, + "total" : { + "total_in_bytes" : 30633005056, + "available_in_bytes" : 11361931264, + "free_in_bytes" : 12989317120 + }, + "timestamp" : 1586197973966, + "data" : [ + { + "path" : "/usr/share/elasticsearch/data/nodes/0", + "type" : "overlay", + "mount" : "/ (overlay)", + "free_in_bytes" : 12989317120, + "total_in_bytes" : 30633005056, + "available_in_bytes" : 11361931264 + } + ], + "io_stats" : {}, + "most_usage_estimate" : { + "used_disk_percent" : 62.9089363866555, + "path" : "/usr/share/elasticsearch/data/nodes/0", + "total_in_bytes" : 30633005056, + "available_in_bytes" : 11362107392 + } + }, + "transport" : { + "tx_count" : 10, + "server_open" : 0, + "rx_size_in_bytes" : 2997, + "tx_size_in_bytes" : 2997, + "rx_count" : 10 + }, + "script" : { + "compilations" : 0, + "cache_evictions" : 0 + }, + "process" : { + "timestamp" : 1586197973965, + "open_file_descriptors" : 231, + "mem" : { + "total_virtual_in_bytes" : 6583902208 + }, + "max_file_descriptors" : 1048576, + "cpu" : { + "total_in_millis" : 26650, + "percent" : 0 + } + }, + "breakers" : { + "request" : { + "tripped" : 0, + "limit_size_in_bytes" : 322122547, + "limit_size" : "307.1mb", + "estimated_size_in_bytes" : 0, + "estimated_size" : "0b", + "overhead" : 1 + }, + "parent" : { + "estimated_size_in_bytes" : 4091, + "limit_size" : "358.3mb", + "limit_size_in_bytes" : 375809638, + "tripped" : 0, + "overhead" : 1, + "estimated_size" : "3.9kb" + }, + "fielddata" : { + "estimated_size_in_bytes" : 0, + "limit_size" : "307.1mb", + "tripped" : 0, + "limit_size_in_bytes" : 322122547, + "overhead" : 1.03, + "estimated_size" : "0b" + }, + "accounting" : { + "estimated_size" : "3.9kb", + "overhead" : 1, + "limit_size" : "512mb", + "limit_size_in_bytes" : 536870912, + "tripped" : 0, + "estimated_size_in_bytes" : 4091 + }, + "in_flight_requests" : { + "estimated_size_in_bytes" : 0, + "limit_size" : "512mb", + "tripped" : 0, + "limit_size_in_bytes" : 536870912, + "overhead" : 1, + "estimated_size" : "0b" + } + }, + "os" : { + "mem" : { + "total_in_bytes" : 33545908224, + "used_in_bytes" : 23597338624, + "free_in_bytes" : 9948569600, + "used_percent" : 70, + "free_percent" : 30 + }, + "cpu" : { + "load_average" : { + "5m" : 2.11, + "15m" : 2.28, + "1m" : 1.44 + }, + "percent" : 11 + }, + "swap" : { + "free_in_bytes" : 667029504, + "total_in_bytes" : 1023406080, + "used_in_bytes" : 356376576 + }, + "timestamp" : 1586197973964 + }, + "thread_pool" : { + "index" : { + "largest" : 0, + "completed" : 0, + "threads" : 0, + "queue" : 0, + "active" : 0, + "rejected" : 0 + }, + "warmer" : { + "queue" : 0, + "active" : 0, + "rejected" : 0, + "threads" : 0, + "completed" : 0, + "largest" : 0 + }, + "flush" : { + "queue" : 0, + "active" : 0, + "rejected" : 0, + "completed" : 0, + "threads" : 0, + "largest" : 0 + }, + "generic" : { + "queue" : 0, + "rejected" : 0, + "active" : 0, + "completed" : 97, + "largest" : 4, + "threads" : 4 + }, + "force_merge" : { + "queue" : 0, + "rejected" : 0, + "active" : 0, + "completed" : 0, + "largest" : 0, + "threads" : 0 + }, + "snapshot" : { + "completed" : 0, + "threads" : 0, + "largest" : 0, + "queue" : 0, + "active" : 0, + "rejected" : 0 + }, + "analyze" : { + "largest" : 0, + "completed" : 0, + "threads" : 0, + "rejected" : 0, + "active" : 0, + "queue" : 0 + }, + "listener" : { + "completed" : 0, + "threads" : 0, + "largest" : 0, + "rejected" : 0, + "active" : 0, + "queue" : 0 + }, + "fetch_shard_started" : { + "completed" : 9, + "largest" : 9, + "threads" : 9, + "queue" : 0, + "rejected" : 0, + "active" : 0 + }, + "get" : { + "queue" : 0, + "rejected" : 0, + "active" : 0, + "largest" : 1, + "completed" : 1, + "threads" : 1 + }, + "fetch_shard_store" : { + "active" : 0, + "rejected" : 0, + "queue" : 0, + "threads" : 0, + "completed" : 0, + "largest" : 0 + }, + "search_throttled" : { + "completed" : 0, + "largest" : 0, + "threads" : 0, + "queue" : 0, + "active" : 0, + "rejected" : 0 + }, + "refresh" : { + "active" : 0, + "rejected" : 0, + "queue" : 0, + "threads" : 3, + "completed" : 1305, + "largest" : 3 + }, + "search" : { + "rejected" : 0, + "active" : 0, + "queue" : 0, + "completed" : 7, + "largest" : 7, + "threads" : 7 + }, + "write" : { + "completed" : 0, + "threads" : 0, + "largest" : 0, + "queue" : 0, + "rejected" : 0, + "active" : 0 + }, + "management" : { + "active" : 1, + "rejected" : 0, + "queue" : 0, + "completed" : 77, + "threads" : 3, + "largest" : 3 + } + }, + "transport_address" : "172.18.0.2:9300" + } + }, + "_nodes" : { + "failed" : 0, + "successful" : 1, + "total" : 1 + }, + "cluster_name" : "docker-cluster" +} From 1182a5059b26c0af94e00228269a01b8ada0f381 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 28 Jul 2021 11:09:08 +0530 Subject: [PATCH 29/37] Fixed bugs in logout flow --- out | 774 ------------------------------------------------------------ 1 file changed, 774 deletions(-) delete mode 100644 out diff --git a/out b/out deleted file mode 100644 index 37f79ce33..000000000 --- a/out +++ /dev/null @@ -1,774 +0,0 @@ -Attaching to elasticsearch, foxtrot_hbase_1, foxtrot_server -hbase_1 | /usr/lib/python2.7/dist-packages/supervisor/options.py:296: UserWarning: Supervisord is running as root and it is searching for its configuration file in default locations (including its current working directory); you probably want to specify a "-c" argument specifying an absolute path to a configuration file for improved security. -hbase_1 | 'Supervisord is running as root and it is searching ' -foxtrot_server | No CONFIG_PATH defined. We shall be using default config from /config/docker.yml -hbase_1 | 2020-07-22 09:39:17,221 CRIT Supervisor running as root (no user in config file) -foxtrot_server | Init sleep of 15 seconds specified. Waiting... -hbase_1 | 2020-07-22 09:39:17,221 WARN Included extra file "/etc/supervisor/conf.d/supervisord.conf" during parsing -hbase_1 | Unlinking stale socket /var/run/supervisor.sock -hbase_1 | 2020-07-22 09:39:17,531 INFO RPC interface 'supervisor' initialized -hbase_1 | 2020-07-22 09:39:17,531 CRIT Server 'unix_http_server' running without any HTTP authentication checking -hbase_1 | 2020-07-22 09:39:17,531 INFO supervisord started with pid 1 -hbase_1 | 2020-07-22 09:39:18,533 INFO spawned: 'stdout' with pid 8 -hbase_1 | 2020-07-22 09:39:18,535 INFO spawned: 'hbase-rest' with pid 9 -hbase_1 | 2020-07-22 09:39:18,536 INFO spawned: 'hbase' with pid 10 -hbase_1 | starting rest, logging to /opt/hbase/bin/../logs/hbase--rest-hbase.out -elasticsearch | [2020-07-22T09:39:19,168][INFO ][o.e.e.NodeEnvironment ] [5q9Ine5] using [1] data paths, mounts [[/ (overlay)]], net usable_space [2.4gb], net total_space [28.5gb], types [overlay] -elasticsearch | [2020-07-22T09:39:19,171][INFO ][o.e.e.NodeEnvironment ] [5q9Ine5] heap size [512mb], compressed ordinary object pointers [true] -elasticsearch | [2020-07-22T09:39:19,200][INFO ][o.e.n.Node ] [5q9Ine5] node name derived from node ID [5q9Ine5JToGzVTBOdMThhw]; set [node.name] to override -elasticsearch | [2020-07-22T09:39:19,200][INFO ][o.e.n.Node ] [5q9Ine5] version[6.8.8], pid[1], build[oss/docker/2f4c224/2020-03-18T23:22:18.622755Z], OS[Linux/5.3.0-62-generic/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/14/14+36] -elasticsearch | [2020-07-22T09:39:19,201][INFO ][o.e.n.Node ] [5q9Ine5] JVM arguments [-Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/tmp/elasticsearch-4753755361850039883, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.locale.providers=COMPAT, -XX:UseAVX=2, -Xms512m, -Xmx512m, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=oss, -Des.distribution.type=docker] -hbase_1 | 2020-07-22 09:39:19,573 INFO success: stdout entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) -hbase_1 | 2020-07-22 09:39:19,573 INFO success: hbase-rest entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) -hbase_1 | 2020-07-22 09:39:19,573 INFO success: hbase entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) -hbase_1 | hbase stderr | OpenJDK 64-Bit Server VM warning: hbase stderr | ignoring option PermSize=128m; support was removed in 8.0 hbase stderr | -hbase_1 | hbase stderr | OpenJDK 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0 -hbase_1 | hbase-rest stdout | starting rest, logging to /opt/hbase/bin/../logs/hbase--rest-hbase.out -hbase_1 | hbase stderr | 2020-07-22 09:39:19,097 INFO [main] util.VersionInfo: HBase 1.2.1 -hbase_1 | hbase stderr | 2020-07-22 09:39:19,098 INFO [main] util.VersionInfo: Source code repository git://asf-dev/home/busbey/projects/hbase revision=8d8a7107dc4ccbf36a92f64675dc60392f85c015 -hbase_1 | 2020-07-22 09:39:19,098 INFO [main] util.VersionInfo: Compiled by busbey on Wed Mar 30 11:19:21 CDT 2016 -hbase_1 | 2020-07-22 09:39:19,098 INFO [main] util.VersionInfo: From source with checksum f4bb4a14bb4e0b72b46f729dae98a772 -hbase_1 | hbase stderr | 2020-07-22 09:39:19,572 INFO [main] master.HMasterCommandLine: Starting a zookeeper cluster -hbase_1 | hbase stderr | 2020-07-22 09:39:19,630 INFO [main] server.ZooKeeperServer: Server environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT -hbase_1 | 2020-07-22 09:39:19,634 INFO exited: hbase-rest (exit status 0; expected) -hbase_1 | hbase stderr | 2020-07-22 09:39:19,634 INFO [main] server.ZooKeeperServer: Server environment:host.name=hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:19,638 INFO [main] server.ZooKeeperServer: Server environment:java.version=1.8.0_72-internal -hbase_1 | hbase stderr | 2020-07-22 09:39:19,639 INFO [main] server.ZooKeeperServer: Server environment:java.vendor=Oracle Corporation -hbase_1 | hbase stderr | 2020-07-22 09:39:19,639 INFO [main] server.ZooKeeperServer: Server environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre -hbase_1 | hbase stderr | 2020-07-22 09:39:19,640 INFO [main] server.ZooKeeperServer: Server environment:java.class.path=/opt/hbase/bin/../conf:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../lib/activation-1.1.jar:/opt/hbase/bin/../lib/antisamy-1.4.3.jar:/opt/hbase/bin/../lib/aopalliance-1.0.jar:/opt/hbase/bin/../lib/apacheds-i18n-2.0.0-M15.jar:/opt/hbase/bin/../lib/apacheds-kerberos-codec-2.0.0-M15.jar:/opt/hbase/bin/../lib/api-asn1-api-1.0.0-M20.jar:/opt/hbase/bin/../lib/api-util-1.0.0-M20.jar:/opt/hbase/bin/../lib/asm-3.1.jar:/opt/hbase/bin/../lib/avro-1.7.4.jar:/opt/hbase/bin/../lib/batik-css-1.7.jar:/opt/hbase/bin/../lib/batik-ext-1.7.jar:/opt/hbase/bin/../lib/batik-util-1.7.jar:/opt/hbase/bin/../lib/bsh-core-2.0b4.jar:/opt/hbase/bin/../lib/commons-beanutils-1.7.0.jar:/opt/hbase/bin/../lib/commons-beanutils-core-1.7.0.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.9.jar:/opt/hbase/bin/../lib/commons-collections-3.2.2.jar:/opt/hbase/bin/../lib/commons-compress-1.4.1.jar:/opt/hbase/bin/../lib/commons-configuration-1.6.jar:/opt/hbase/bin/../lib/commons-daemon-1.0.13.jar:/opt/hbase/bin/../lib/commons-digester-1.8.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-fileupload-1.2.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-io-2.4.jar:/opt/hbase/bin/../lib/commons-lang-2.6.jar:/opt/hbase/bin/../lib/commons-logging-1.2.jar:/opt/hbase/bin/../lib/commons-math-2.2.jar:/opt/hbase/bin/../lib/commons-math3-3.1.1.jar:/opt/hbase/bin/../lib/commons-net-3.1.jar:/opt/hbase/bin/../lib/disruptor-3.3.0.jar:/opt/hbase/bin/../lib/esapi-2.1.0.jar:/opt/hbase/bin/../lib/findbugs-annotations-1.3.9-1.jar:/opt/hbase/bin/../lib/guava-12.0.1.jar:/opt/hbase/bin/../lib/guice-3.0.jar:/opt/hbase/bin/../lib/guice-servlet-3.0.jar:/opt/hbase/bin/../lib/hadoop-annotations-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-auth-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-client-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-hdfs-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-app-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-core-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-jobclient-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-shuffle-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-api-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-client-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-server-common-2.5.1.jar:/opt/hbase/bin/../lib/hbase-annotations-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-annotations-1.2.1.jar:/opt/hbase/bin/../lib/hbase-client-1.2.1.jar:/opt/hbase/bin/../lib/hbase-common-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-common-1.2.1.jar:/opt/hbase/bin/../lib/hbase-examples-1.2.1.jar:/opt/hbase/bin/../lib/hbase-external-blockcache-1.2.1.jar:/opt/hbase/bin/../lib/hbase-hadoop-compat-1.2.1.jar:/opt/hbase/bin/../lib/hbase-hadoop2-compat-1.2.1.jar:/opt/hbase/bin/../lib/hbase-it-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-it-1.2.1.jar:/opt/hbase/bin/../lib/hbase-prefix-tree-1.2.1.jar:/opt/hbase/bin/../lib/hbase-procedure-1.2.1.jar:/opt/hbase/bin/../lib/hbase-protocol-1.2.1.jar:/opt/hbase/bin/../lib/hbase-resource-bundle-1.2.1.jar:/opt/hbase/bin/../lib/hbase-rest-1.2.1.jar:/opt/hbase/bin/../lib/hbase-server-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-server-1.2.1.jar:/opt/hbase/bin/../lib/hbase-shell-1.2.1.jar:/opt/hbase/bin/../lib/hbase-thrift-1.2.1.jar:/opt/hbase/bin/../lib/htrace-core-3.1.0-incubating.jar:/opt/hbase/bin/../lib/httpclient-4.2.5.jar:/opt/hbase/bin/../lib/httpcore-4.4.1.jar:/opt/hbase/bin/../lib/jackson-core-asl-1.9.13.jar:/opt/hbase/bin/../lib/jackson-jaxrs-1.9.13.jar:/opt/hbase/bin/../lib/jackson-mapper-asl-1.9.13.jar:/opt/hbase/bin/../lib/jackson-xc-1.9.13.jar:/opt/hbase/bin/../lib/jamon-runtime-2.4.1.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/java-xmlbuilder-0.4.jar:/opt/hbase/bin/../lib/javax.inject-1.jar:/opt/hbase/bin/../lib/jaxb-api-2.2.2.jar:/opt/hbase/bin/../lib/jaxb-impl-2.2.3-1.jar:/opt/hbase/bin/../lib/jcodings-1.0.8.jar:/opt/hbase/bin/../lib/jersey-client-1.9.jar:/opt/hbase/bin/../lib/jersey-core-1.9.jar:/opt/hbase/bin/../lib/jersey-guice-1.9.jar:/opt/hbase/bin/../lib/jersey-json-1.9.jar:/opt/hbase/bin/../lib/jersey-server-1.9.jar:/opt/hbase/bin/../lib/jets3t-0.9.0.jar:/opt/hbase/bin/../lib/jettison-1.3.3.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-sslengine-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/joni-2.1.2.jar:/opt/hbase/bin/../lib/jruby-complete-1.6.8.jar:/opt/hbase/bin/../lib/jsch-0.1.42.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr305-1.3.9.jar:/opt/hbase/bin/../lib/junit-4.12.jar:/opt/hbase/bin/../lib/leveldbjni-all-1.8.jar:/opt/hbase/bin/../lib/libthrift-0.9.3.jar:/opt/hbase/bin/../lib/log4j-1.2.17.jar:/opt/hbase/bin/../lib/metrics-core-2.2.0.jar:/opt/hbase/bin/../lib/nekohtml-1.9.12.jar:/opt/hbase/bin/../lib/netty-all-4.0.23.Final.jar:/opt/hbase/bin/../lib/paranamer-2.3.jar:/opt/hbase/bin/../lib/protobuf-java-2.5.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/servlet-api-2.5.jar:/opt/hbase/bin/../lib/slf4j-api-1.7.7.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.7.5.jar:/opt/hbase/bin/../lib/snappy-java-1.0.4.1.jar:/opt/hbase/bin/../lib/spymemcached-2.11.6.jar:/opt/hbase/bin/../lib/xalan-2.7.0.jar:/opt/hbase/bin/../lib/xml-apis-1.3.03.jar:/opt/hbase/bin/../lib/xml-apis-ext-1.3.04.jar:/opt/hbase/bin/../lib/xmlenc-0.52.jar:/opt/hbase/bin/../lib/xom-1.2.5.jar:/opt/hbase/bin/../lib/xz-1.0.jar:/opt/hbase/bin/../lib/zookeeper-3.4.6.jar: -hbase_1 | hbase stderr | 2020-07-22 09:39:19,640 INFO [main] server.ZooKeeperServer: Server environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib -hbase_1 | hbase stderr | 2020-07-22 09:39:19,641 INFO [main] server.ZooKeeperServer: Server environment:java.io.tmpdir=/tmp -hbase_1 | hbase stderr | 2020-07-22 09:39:19,641 INFO [main] server.ZooKeeperServer: Server environment:java.compiler= -hbase_1 | hbase stderr | 2020-07-22 09:39:19,642 INFO [main] server.ZooKeeperServer: Server environment:os.name=Linux -hbase_1 | hbase stderr | 2020-07-22 09:39:19,642 INFO [main] server.ZooKeeperServer: Server environment:os.arch=amd64 -hbase_1 | hbase stderr | 2020-07-22 09:39:19,642 INFO [main] server.ZooKeeperServer: Server environment:os.version=5.3.0-62-generic -hbase_1 | hbase stderr | 2020-07-22 09:39:19,643 INFO [main] server.ZooKeeperServer: Server environment:user.name=root -hbase_1 | hbase stderr | 2020-07-22 09:39:19,643 INFO [main] server.ZooKeeperServer: Server environment:user.home=/root -hbase_1 | hbase stderr | 2020-07-22 09:39:19,644 INFO [main] server.ZooKeeperServer: Server environment:user.dir=/opt -hbase_1 | hbase stderr | 2020-07-22 09:39:19,695 INFO [main] server.ZooKeeperServer: Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 datadir /data/zookeeper/zookeeper_0/version-2 snapdir /data/zookeeper/zookeeper_0/version-2 -hbase_1 | hbase stderr | 2020-07-22 09:39:19,712 INFO [main] server.NIOServerCnxnFactory: binding to port 0.0.0.0/0.0.0.0:2181 -hbase_1 | hbase stderr | 2020-07-22 09:39:19,731 INFO [main] persistence.FileSnap: Reading snapshot /data/zookeeper/zookeeper_0/version-2/snapshot.1ec33 -elasticsearch | [2020-07-22T09:39:20,627][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [aggs-matrix-stats] -elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [analysis-common] -elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [ingest-common] -elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [ingest-geoip] -elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [ingest-user-agent] -elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [lang-expression] -elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [lang-mustache] -elasticsearch | [2020-07-22T09:39:20,631][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [lang-painless] -elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [mapper-extras] -elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [parent-join] -elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [percolator] -elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [rank-eval] -elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [reindex] -elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [repository-url] -elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [transport-netty4] -elasticsearch | [2020-07-22T09:39:20,632][INFO ][o.e.p.PluginsService ] [5q9Ine5] loaded module [tribe] -elasticsearch | [2020-07-22T09:39:20,633][INFO ][o.e.p.PluginsService ] [5q9Ine5] no plugins loaded -hbase_1 | hbase stderr | 2020-07-22 09:39:20,371 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34206 -hbase_1 | hbase stderr | 2020-07-22 09:39:20,383 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxn: Processing stat command from /127.0.0.1:34206 -hbase_1 | hbase stderr | 2020-07-22 09:39:20,395 INFO [Thread-2] server.NIOServerCnxn: Stat command output -hbase_1 | hbase stderr | 2020-07-22 09:39:20,396 INFO [Thread-2] server.NIOServerCnxn: Closed socket connection for client /127.0.0.1:34206 (no session established for client) -hbase_1 | hbase stderr | 2020-07-22 09:39:20,396 INFO [main] zookeeper.MiniZooKeeperCluster: Started MiniZooKeeperCluster and ran successful 'stat' on client port=2181 -hbase_1 | hbase stderr | 2020-07-22 09:39:20,397 INFO [main] master.HMasterCommandLine: Starting up instance of localHBaseCluster; master=1, regionserversCount=1 -hbase_1 | hbase stderr | 2020-07-22 09:39:20,794 WARN [main] util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -hbase_1 | hbase stderr | 2020-07-22 09:39:21,373 INFO [main] regionserver.RSRpcServices: master/hbase/172.18.0.2:0 server-side HConnection retries=350 -hbase_1 | hbase stderr | 2020-07-22 09:39:21,604 INFO [main] ipc.SimpleRpcScheduler: Using deadline as user call queue, count=3 -hbase_1 | hbase stderr | 2020-07-22 09:39:21,624 INFO [main] ipc.RpcServer: master/hbase/172.18.0.2:0: started 10 reader(s) listening on port=41229 -hbase_1 | hbase stderr | 2020-07-22 09:39:21,694 INFO [main] impl.MetricsConfig: loaded properties from hadoop-metrics2-hbase.properties -hbase_1 | hbase stderr | 2020-07-22 09:39:21,719 INFO [main] impl.MetricsSystemImpl: Scheduled snapshot period at 10 second(s). -hbase_1 | 2020-07-22 09:39:21,719 INFO [main] impl.MetricsSystemImpl: HBase metrics system started -hbase_1 | hbase stderr | 2020-07-22 09:39:21,995 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=master:41229 connecting to ZooKeeper ensemble=localhost:2181 -hbase_1 | hbase stderr | 2020-07-22 09:39:22,000 INFO [main] zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT -hbase_1 | hbase stderr | 2020-07-22 09:39:22,000 INFO [main] zookeeper.ZooKeeper: Client environment:host.name=hbase -hbase_1 | 2020-07-22 09:39:22,000 INFO [main] zookeeper.ZooKeeper: Client environment:java.version=1.8.0_72-internal -hbase_1 | 2020-07-22 09:39:22,000 INFO [main] zookeeper.ZooKeeper: Client environment:java.vendor=Oracle Corporation -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre -hbase_1 | hbase stderr | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.class.path=/opt/hbase/bin/../conf:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../lib/activation-1.1.jar:/opt/hbase/bin/../lib/antisamy-1.4.3.jar:/opt/hbase/bin/../lib/aopalliance-1.0.jar:/opt/hbase/bin/../lib/apacheds-i18n-2.0.0-M15.jar:/opt/hbase/bin/../lib/apacheds-kerberos-codec-2.0.0-M15.jar:/opt/hbase/bin/../lib/api-asn1-api-1.0.0-M20.jar:/opt/hbase/bin/../lib/api-util-1.0.0-M20.jar:/opt/hbase/bin/../lib/asm-3.1.jar:/opt/hbase/bin/../lib/avro-1.7.4.jar:/opt/hbase/bin/../lib/batik-css-1.7.jar:/opt/hbase/bin/../lib/batik-ext-1.7.jar:/opt/hbase/bin/../lib/batik-util-1.7.jar:/opt/hbase/bin/../lib/bsh-core-2.0b4.jar:/opt/hbase/bin/../lib/commons-beanutils-1.7.0.jar:/opt/hbase/bin/../lib/commons-beanutils-core-1.7.0.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.9.jar:/opt/hbase/bin/../lib/commons-collections-3.2.2.jar:/opt/hbase/bin/../lib/commons-compress-1.4.1.jar:/opt/hbase/bin/../lib/commons-configuration-1.6.jar:/opt/hbase/bin/../lib/commons-daemon-1.0.13.jar:/opt/hbase/bin/../lib/commons-digester-1.8.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-fileupload-1.2.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-io-2.4.jar:/opt/hbase/bin/../lib/commons-lang-2.6.jar:/opt/hbase/bin/../lib/commons-logging-1.2.jar:/opt/hbase/bin/../lib/commons-math-2.2.jar:/opt/hbase/bin/../lib/commons-math3-3.1.1.jar:/opt/hbase/bin/../lib/commons-net-3.1.jar:/opt/hbase/bin/../lib/disruptor-3.3.0.jar:/opt/hbase/bin/../lib/esapi-2.1.0.jar:/opt/hbase/bin/../lib/findbugs-annotations-1.3.9-1.jar:/opt/hbase/bin/../lib/guava-12.0.1.jar:/opt/hbase/bin/../lib/guice-3.0.jar:/opt/hbase/bin/../lib/guice-servlet-3.0.jar:/opt/hbase/bin/../lib/hadoop-annotations-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-auth-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-client-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-hdfs-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-app-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-core-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-jobclient-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-mapreduce-client-shuffle-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-api-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-client-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-common-2.5.1.jar:/opt/hbase/bin/../lib/hadoop-yarn-server-common-2.5.1.jar:/opt/hbase/bin/../lib/hbase-annotations-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-annotations-1.2.1.jar:/opt/hbase/bin/../lib/hbase-client-1.2.1.jar:/opt/hbase/bin/../lib/hbase-common-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-common-1.2.1.jar:/opt/hbase/bin/../lib/hbase-examples-1.2.1.jar:/opt/hbase/bin/../lib/hbase-external-blockcache-1.2.1.jar:/opt/hbase/bin/../lib/hbase-hadoop-compat-1.2.1.jar:/opt/hbase/bin/../lib/hbase-hadoop2-compat-1.2.1.jar:/opt/hbase/bin/../lib/hbase-it-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-it-1.2.1.jar:/opt/hbase/bin/../lib/hbase-prefix-tree-1.2.1.jar:/opt/hbase/bin/../lib/hbase-procedure-1.2.1.jar:/opt/hbase/bin/../lib/hbase-protocol-1.2.1.jar:/opt/hbase/bin/../lib/hbase-resource-bundle-1.2.1.jar:/opt/hbase/bin/../lib/hbase-rest-1.2.1.jar:/opt/hbase/bin/../lib/hbase-server-1.2.1-tests.jar:/opt/hbase/bin/../lib/hbase-server-1.2.1.jar:/opt/hbase/bin/../lib/hbase-shell-1.2.1.jar:/opt/hbase/bin/../lib/hbase-thrift-1.2.1.jar:/opt/hbase/bin/../lib/htrace-core-3.1.0-incubating.jar:/opt/hbase/bin/../lib/httpclient-4.2.5.jar:/opt/hbase/bin/../lib/httpcore-4.4.1.jar:/opt/hbase/bin/../lib/jackson-core-asl-1.9.13.jar:/opt/hbase/bin/../lib/jackson-jaxrs-1.9.13.jar:/opt/hbase/bin/../lib/jackson-mapper-asl-1.9.13.jar:/opt/hbase/bin/../lib/jackson-xc-1.9.13.jar:/opt/hbase/bin/../lib/jamon-runtime-2.4.1.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/java-xmlbuilder-0.4.jar:/opt/hbase/bin/../lib/javax.inject-1.jar:/opt/hbase/bin/../lib/jaxb-api-2.2.2.jar:/opt/hbase/bin/../lib/jaxb-impl-2.2.3-1.jar:/opt/hbase/bin/../lib/jcodings-1.0.8.jar:/opt/hbase/bin/../lib/jersey-client-1.9.jar:/opt/hbase/bin/../lib/jersey-core-1.9.jar:/opt/hbase/bin/../lib/jersey-guice-1.9.jar:/opt/hbase/bin/../lib/jersey-json-1.9.jar:/opt/hbase/bin/../lib/jersey-server-1.9.jar:/opt/hbase/bin/../lib/jets3t-0.9.0.jar:/opt/hbase/bin/../lib/jettison-1.3.3.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-sslengine-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/joni-2.1.2.jar:/opt/hbase/bin/../lib/jruby-complete-1.6.8.jar:/opt/hbase/bin/../lib/jsch-0.1.42.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr305-1.3.9.jar:/opt/hbase/bin/../lib/junit-4.12.jar:/opt/hbase/bin/../lib/leveldbjni-all-1.8.jar:/opt/hbase/bin/../lib/libthrift-0.9.3.jar:/opt/hbase/bin/../lib/log4j-1.2.17.jar:/opt/hbase/bin/../lib/metrics-core-2.2.0.jar:/opt/hbase/bin/../lib/nekohtml-1.9.12.jar:/opt/hbase/bin/../lib/netty-all-4.0.23.Final.jar:/opt/hbase/bin/../lib/paranamer-2.3.jar:/opt/hbase/bin/../lib/protobuf-java-2.5.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/servlet-api-2.5.jar:/opt/hbase/bin/../lib/slf4j-api-1.7.7.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.7.5.jar:/opt/hbase/bin/../lib/snappy-java-1.0.4.1.jar:/opt/hbase/bin/../lib/spymemcached-2.11.6.jar:/opt/hbase/bin/../lib/xalan-2.7.0.jar:/opt/hbase/bin/../lib/xml-apis-1.3.03.jar:/opt/hbase/bin/../lib/xml-apis-ext-1.3.04.jar:/opt/hbase/bin/../lib/xmlenc-0.52.jar:/opt/hbase/bin/../lib/xom-1.2.5.jar:/opt/hbase/bin/../lib/xz-1.0.jar:/opt/hbase/bin/../lib/zookeeper-3.4.6.jar: -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:java.compiler= -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:os.name=Linux -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:os.arch=amd64 -hbase_1 | hbase stderr | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:os.version=5.3.0-62-generic -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:user.name=root -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:user.home=/root -hbase_1 | 2020-07-22 09:39:22,001 INFO [main] zookeeper.ZooKeeper: Client environment:user.dir=/opt -hbase_1 | hbase stderr | 2020-07-22 09:39:22,002 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=master:412290x0, quorum=localhost:2181, baseZNode=/hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:22,020 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) -hbase_1 | hbase stderr | 2020-07-22 09:39:22,020 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34218 -hbase_1 | hbase stderr | 2020-07-22 09:39:22,021 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session -hbase_1 | hbase stderr | 2020-07-22 09:39:22,022 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34218 -hbase_1 | hbase stderr | 2020-07-22 09:39:22,030 INFO [SyncThread:0] persistence.FileTxnLog: Creating new log file: log.2100c -hbase_1 | hbase stderr | 2020-07-22 09:39:22,041 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0000 with negotiated timeout 10000 for client /127.0.0.1:34218 -hbase_1 | hbase stderr | 2020-07-22 09:39:22,042 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0000, negotiated timeout = 10000 -hbase_1 | hbase stderr | 2020-07-22 09:39:22,046 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x1 zxid:0x2100d txntype:-1 reqpath:n/a Error Path:/hbase Error:KeeperErrorCode = NodeExists for /hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:22,086 INFO [RpcServer.responder] ipc.RpcServer: RpcServer.responder: starting -hbase_1 | hbase stderr | 2020-07-22 09:39:22,090 INFO [RpcServer.listener,port=41229] ipc.RpcServer: RpcServer.listener,port=41229: starting -hbase_1 | hbase stderr | 2020-07-22 09:39:22,328 INFO [main] mortbay.log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog -hbase_1 | hbase stderr | 2020-07-22 09:39:22,333 INFO [main] http.HttpRequestLog: Http request log for http.requests.master is not defined -hbase_1 | hbase stderr | 2020-07-22 09:39:22,347 INFO [main] http.HttpServer: Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) -hbase_1 | hbase stderr | 2020-07-22 09:39:22,347 INFO [main] http.HttpServer: Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) -hbase_1 | hbase stderr | 2020-07-22 09:39:22,350 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master -hbase_1 | hbase stderr | 2020-07-22 09:39:22,350 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static -hbase_1 | 2020-07-22 09:39:22,350 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs -hbase_1 | hbase stderr | 2020-07-22 09:39:22,377 INFO [main] http.HttpServer: Jetty bound to port 16010 -hbase_1 | 2020-07-22 09:39:22,377 INFO [main] mortbay.log: jetty-6.1.26 -hbase_1 | hbase stderr | 2020-07-22 09:39:22,843 INFO [main] mortbay.log: Started SelectChannelConnector@0.0.0.0:16010 -hbase_1 | hbase stderr | 2020-07-22 09:39:22,853 INFO [main] master.HMaster: hbase.rootdir=file:/data/hbase, hbase.cluster.distributed=false -hbase_1 | hbase stderr | 2020-07-22 09:39:22,877 INFO [main] master.HMaster: Adding backup master ZNode /hbase/backup-masters/hbase,41229,1595410761736 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,033 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x10 zxid:0x2100f txntype:-1 reqpath:n/a Error Path:/hbase/master Error:KeeperErrorCode = NodeExists for /hbase/master -hbase_1 | hbase stderr | 2020-07-22 09:39:23,053 INFO [hbase:41229.activeMasterManager] master.ActiveMasterManager: Another master is the active master, hbase,41223,1595410680501; waiting to become the next active master -hbase_1 | hbase stderr | 2020-07-22 09:39:23,085 INFO [main] regionserver.RSRpcServices: regionserver/hbase/172.18.0.2:0 server-side HConnection retries=350 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,086 INFO [main] ipc.SimpleRpcScheduler: Using deadline as user call queue, count=3 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,147 INFO [main] ipc.RpcServer: regionserver/hbase/172.18.0.2:0: started 10 reader(s) listening on port=41633 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,149 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=regionserver:41633 connecting to ZooKeeper ensemble=localhost:2181 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,151 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=regionserver:416330x0, quorum=localhost:2181, baseZNode=/hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:23,158 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) -hbase_1 | hbase stderr | 2020-07-22 09:39:23,158 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34220 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,207 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session -hbase_1 | hbase stderr | 2020-07-22 09:39:23,208 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34220 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,219 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0001 with negotiated timeout 10000 for client /127.0.0.1:34220 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,219 INFO [main-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0001, negotiated timeout = 10000 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,227 INFO [RpcServer.responder] ipc.RpcServer: RpcServer.responder: starting -hbase_1 | hbase stderr | 2020-07-22 09:39:23,234 INFO [RpcServer.listener,port=41633] ipc.RpcServer: RpcServer.listener,port=41633: starting -hbase_1 | hbase stderr | 2020-07-22 09:39:23,298 INFO [main] http.HttpRequestLog: Http request log for http.requests.regionserver is not defined -hbase_1 | hbase stderr | 2020-07-22 09:39:23,299 INFO [main] http.HttpServer: Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) -hbase_1 | hbase stderr | 2020-07-22 09:39:23,299 INFO [main] http.HttpServer: Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) -elasticsearch | [2020-07-22T09:39:24,622][INFO ][o.e.d.DiscoveryModule ] [5q9Ine5] using discovery type [zen] and host providers [settings] -elasticsearch | [2020-07-22T09:39:25,037][INFO ][o.e.n.Node ] [5q9Ine5] initialized -elasticsearch | [2020-07-22T09:39:25,037][INFO ][o.e.n.Node ] [5q9Ine5] starting ... -elasticsearch | [2020-07-22T09:39:25,175][INFO ][o.e.t.TransportService ] [5q9Ine5] publish_address {172.18.0.3:9300}, bound_addresses {0.0.0.0:9300} -elasticsearch | [2020-07-22T09:39:25,196][INFO ][o.e.b.BootstrapChecks ] [5q9Ine5] bound or publishing to a non-loopback address, enforcing bootstrap checks -elasticsearch | [2020-07-22T09:39:28,230][INFO ][o.e.c.s.MasterService ] [5q9Ine5] zen-disco-elected-as-master ([0] nodes joined), reason: new_master {5q9Ine5}{5q9Ine5JToGzVTBOdMThhw}{_w0q-7V6SiWl02g_-m4GQA}{172.18.0.3}{172.18.0.3:9300} -elasticsearch | [2020-07-22T09:39:28,233][INFO ][o.e.c.s.ClusterApplierService] [5q9Ine5] new_master {5q9Ine5}{5q9Ine5JToGzVTBOdMThhw}{_w0q-7V6SiWl02g_-m4GQA}{172.18.0.3}{172.18.0.3:9300}, reason: apply cluster state (from master [master {5q9Ine5}{5q9Ine5JToGzVTBOdMThhw}{_w0q-7V6SiWl02g_-m4GQA}{172.18.0.3}{172.18.0.3:9300} committed version [1] source [zen-disco-elected-as-master ([0] nodes joined)]]) -elasticsearch | [2020-07-22T09:39:28,243][INFO ][o.e.h.n.Netty4HttpServerTransport] [5q9Ine5] publish_address {172.18.0.3:9200}, bound_addresses {0.0.0.0:9200} -elasticsearch | [2020-07-22T09:39:28,243][INFO ][o.e.n.Node ] [5q9Ine5] started -elasticsearch | [2020-07-22T09:39:28,386][INFO ][o.e.g.GatewayService ] [5q9Ine5] recovered [7] indices into cluster_state -elasticsearch | [2020-07-22T09:39:28,659][INFO ][o.e.c.r.a.AllocationService] [5q9Ine5] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[consoles_history][0], [table-meta][0], [consoles_v2][0], [consoles][0]] ...]). -hbase_1 | hbase stderr | 2020-07-22 09:39:23,300 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver -hbase_1 | hbase stderr | 2020-07-22 09:39:23,300 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs -hbase_1 | 2020-07-22 09:39:23,300 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static -hbase_1 | hbase stderr | 2020-07-22 09:39:23,303 INFO [main] http.HttpServer: Jetty bound to port 36787 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,303 INFO [main] mortbay.log: jetty-6.1.26 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,427 INFO [main] mortbay.log: Started SelectChannelConnector@0.0.0.0:36787 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,519 INFO [M:0;hbase:41229] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x33489efd connecting to ZooKeeper ensemble=localhost:2181 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,519 INFO [M:0;hbase:41229] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=hconnection-0x33489efd0x0, quorum=localhost:2181, baseZNode=/hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:23,523 INFO [M:0;hbase:41229-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) -hbase_1 | hbase stderr | 2020-07-22 09:39:23,523 INFO [M:0;hbase:41229-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session -hbase_1 | hbase stderr | 2020-07-22 09:39:23,526 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34222 -hbase_1 | 2020-07-22 09:39:23,528 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34222 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,530 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0002 with negotiated timeout 10000 for client /127.0.0.1:34222 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,530 INFO [M:0;hbase:41229-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0002, negotiated timeout = 10000 -hbase_1 | hbase stderr | 2020-07-22 09:39:23,563 INFO [M:0;hbase:41229] regionserver.HRegionServer: ClusterId : 0cbe9efc-c509-45a5-97a4-499957bb7b66 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770005, timeout of 10000ms exceeded -hbase_1 | hbase stderr | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770000, timeout of 10000ms exceeded -hbase_1 | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770004, timeout of 10000ms exceeded -hbase_1 | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770002, timeout of 10000ms exceeded -hbase_1 | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770003, timeout of 10000ms exceeded -hbase_1 | 2020-07-22 09:39:32,001 INFO [SessionTracker] server.ZooKeeperServer: Expiring session 0x17375e303770001, timeout of 10000ms exceeded -hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770005 -hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770000 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770004 -hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770002 -hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770003 -hbase_1 | 2020-07-22 09:39:32,001 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e303770001 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,010 INFO [hbase:41229.activeMasterManager] master.ActiveMasterManager: Deleting ZNode for /hbase/backup-masters/hbase,41229,1595410761736 from backup master directory -hbase_1 | hbase stderr | 2020-07-22 09:39:32,013 WARN [hbase:41229.activeMasterManager] hbase.ZNodeClearer: Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) -hbase_1 | 2020-07-22 09:39:32,013 INFO [hbase:41229.activeMasterManager] master.ActiveMasterManager: Registered Active Master=hbase,41229,1595410761736 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,056 INFO [hbase:41229.activeMasterManager] coordination.SplitLogManagerCoordination: Found 0 orphan tasks and 0 rescan nodes -hbase_1 | hbase stderr | 2020-07-22 09:39:32,060 INFO [RS:0;hbase:41633] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x561d3720 connecting to ZooKeeper ensemble=localhost:2181 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,060 INFO [RS:0;hbase:41633] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=hconnection-0x561d37200x0, quorum=localhost:2181, baseZNode=/hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:32,065 INFO [RS:0;hbase:41633-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) -hbase_1 | hbase stderr | 2020-07-22 09:39:32,066 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34328 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,066 INFO [RS:0;hbase:41633-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session -hbase_1 | hbase stderr | 2020-07-22 09:39:32,067 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34328 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,068 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0003 with negotiated timeout 10000 for client /127.0.0.1:34328 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,068 INFO [RS:0;hbase:41633-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0003, negotiated timeout = 10000 -hbase_1 | 2020-07-22 09:39:32,069 INFO [hbase:41229.activeMasterManager] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x445dd583 connecting to ZooKeeper ensemble=localhost:2181 -hbase_1 | 2020-07-22 09:39:32,069 INFO [hbase:41229.activeMasterManager] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=hconnection-0x445dd5830x0, quorum=localhost:2181, baseZNode=/hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:32,070 INFO [RS:0;hbase:41633] regionserver.HRegionServer: ClusterId : 0cbe9efc-c509-45a5-97a4-499957bb7b66 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,075 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0001 type:create cxid:0x9 zxid:0x2101c txntype:-1 reqpath:n/a Error Path:/hbase/flush-table-proc/acquired Error:KeeperErrorCode = NodeExists for /hbase/flush-table-proc/acquired -hbase_1 | hbase stderr | 2020-07-22 09:39:32,079 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0001 type:create cxid:0xc zxid:0x2101d txntype:-1 reqpath:n/a Error Path:/hbase/online-snapshot/acquired Error:KeeperErrorCode = NodeExists for /hbase/online-snapshot/acquired -hbase_1 | hbase stderr | 2020-07-22 09:39:32,085 INFO [RS:0;hbase:41633] regionserver.MemStoreFlusher: globalMemStoreLimit=3.1 G, globalMemStoreLimitLowMark=2.9 G, maxHeap=7.7 G -hbase_1 | hbase stderr | 2020-07-22 09:39:32,086 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) -hbase_1 | hbase stderr | 2020-07-22 09:39:32,087 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34330 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,087 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session -hbase_1 | hbase stderr | 2020-07-22 09:39:32,087 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34330 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,089 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0004 with negotiated timeout 10000 for client /127.0.0.1:34330 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,089 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0004, negotiated timeout = 10000 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,090 INFO [RS:0;hbase:41633] regionserver.HRegionServer: CompactionChecker runs every 10sec -hbase_1 | hbase stderr | 2020-07-22 09:39:32,098 INFO [hbase:41229.activeMasterManager] balancer.StochasticLoadBalancer: loading config -hbase_1 | hbase stderr | 2020-07-22 09:39:32,106 INFO [RS:0;hbase:41633] regionserver.RegionServerCoprocessorHost: System coprocessor loading is enabled -hbase_1 | 2020-07-22 09:39:32,106 INFO [RS:0;hbase:41633] regionserver.RegionServerCoprocessorHost: Table coprocessor loading is enabled -hbase_1 | hbase stderr | 2020-07-22 09:39:32,107 INFO [RS:0;hbase:41633] regionserver.HRegionServer: reportForDuty to master=hbase,41229,1595410761736 with port=41633, startcode=1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,187 INFO [hbase:41229.activeMasterManager] master.HMaster: Server active/primary master=hbase,41229,1595410761736, sessionid=0x17375e441fa0000, setting cluster-up flag (Was=true) -hbase_1 | hbase stderr | 2020-07-22 09:39:32,196 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x2f zxid:0x21020 txntype:-1 reqpath:n/a Error Path:/hbase/flush-table-proc/acquired Error:KeeperErrorCode = NodeExists for /hbase/flush-table-proc/acquired -hbase_1 | hbase stderr | 2020-07-22 09:39:32,198 INFO [hbase:41229.activeMasterManager] procedure.ZKProcedureUtil: Clearing all procedure znodes: /hbase/flush-table-proc/acquired /hbase/flush-table-proc/reached /hbase/flush-table-proc/abort -hbase_1 | hbase stderr | 2020-07-22 09:39:32,201 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x35 zxid:0x21021 txntype:-1 reqpath:n/a Error Path:/hbase/online-snapshot/acquired Error:KeeperErrorCode = NodeExists for /hbase/online-snapshot/acquired -hbase_1 | hbase stderr | 2020-07-22 09:39:32,202 INFO [hbase:41229.activeMasterManager] procedure.ZKProcedureUtil: Clearing all procedure znodes: /hbase/online-snapshot/acquired /hbase/online-snapshot/reached /hbase/online-snapshot/abort -hbase_1 | hbase stderr | 2020-07-22 09:39:32,222 INFO [hbase:41229.activeMasterManager] master.MasterCoprocessorHost: System coprocessor loading is enabled -hbase_1 | hbase stderr | 2020-07-22 09:39:32,234 INFO [hbase:41229.activeMasterManager] procedure2.ProcedureExecutor: Starting procedure executor threads=9 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,235 INFO [hbase:41229.activeMasterManager] wal.WALProcedureStore: Starting WAL Procedure Store lease recovery -hbase_1 | hbase stderr | 2020-07-22 09:39:32,237 WARN [hbase:41229.activeMasterManager] wal.WALProcedureStore: Remove uninitialized log: DeprecatedRawLocalFileStatus{path=file:/data/hbase/MasterProcWALs/state-00000000000000000219.log; isDirectory=false; length=0; replication=1; blocksize=33554432; modification_time=1595410713000; access_time=0; owner=; group=; permission=rw-rw-rw-; isSymlink=false} -foxtrot_server | done -foxtrot_server | Executing Init Command: java -jar server.jar initialize /config/docker.yml -foxtrot_server | ERROR StatusLogger Log4j2 could not find a logging implementation. Please add log4j-core to the classpath. Using SimpleLogger to log to the console... -foxtrot_server | INFO [2020-07-22 09:39:34,394] [main] [ElasticsearchConnection]: Starting ElasticSearch Client -foxtrot_server | INFO [2020-07-22 09:39:34,657] [main] [ElasticsearchConnection]: Started ElasticSearch Client -foxtrot_server | INFO [2020-07-22 09:39:34,804] [main] [InitializerCommand]: # data nodes: 1, Setting replica count to: 0 -foxtrot_server | INFO [2020-07-22 09:39:34,823] [main] [InitializerCommand]: Index consoles already exists. Nothing to do. -foxtrot_server | INFO [2020-07-22 09:39:34,830] [main] [InitializerCommand]: Index consoles_v2 already exists. Nothing to do. -foxtrot_server | INFO [2020-07-22 09:39:34,834] [main] [InitializerCommand]: Index table-meta already exists. Nothing to do. -foxtrot_server | INFO [2020-07-22 09:39:34,838] [main] [InitializerCommand]: Index consoles_history already exists. Nothing to do. -foxtrot_server | INFO [2020-07-22 09:39:34,842] [main] [InitializerCommand]: Index fql-store already exists. Nothing to do. -foxtrot_server | INFO [2020-07-22 09:39:34,846] [main] [InitializerCommand]: Index user-meta already exists. Nothing to do. -foxtrot_server | INFO [2020-07-22 09:39:34,850] [main] [InitializerCommand]: Index tokens already exists. Nothing to do. -foxtrot_server | INFO [2020-07-22 09:39:34,850] [main] [InitializerCommand]: Creating mapping -elasticsearch | [2020-07-22T09:39:34,960][INFO ][o.e.c.m.MetaDataIndexTemplateService] [5q9Ine5] adding template [template_foxtrot_mappings] for index patterns [foxtrot-*] -foxtrot_server | INFO [2020-07-22 09:39:34,966] [main] [InitializerCommand]: Created mapping: true -foxtrot_server | INFO [2020-07-22 09:39:34,966] [main] [ElasticsearchConnection]: Stopping ElasticSearch client -foxtrot_server | INFO [2020-07-22 09:39:34,968] [main] [InitializerCommand]: Creating hbase table -foxtrot_server | WARN [2020-07-22 09:39:35,221] [main] [NativeCodeLoader]: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -hbase_1 | hbase stderr | 2020-07-22 09:39:32,246 INFO [hbase:41229.activeMasterManager] wal.WALProcedureStore: Lease acquired for flushLogId: 220 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,260 INFO [hbase:41229.activeMasterManager] zookeeper.RecoverableZooKeeper: Process identifier=replicationLogCleaner connecting to ZooKeeper ensemble=localhost:2181 -hbase_1 | 2020-07-22 09:39:32,260 INFO [hbase:41229.activeMasterManager] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=10000 watcher=replicationLogCleaner0x0, quorum=localhost:2181, baseZNode=/hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:32,263 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) -hbase_1 | hbase stderr | 2020-07-22 09:39:32,263 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /127.0.0.1:34332 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,264 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/127.0.0.1:2181, initiating session -hbase_1 | hbase stderr | 2020-07-22 09:39:32,265 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /127.0.0.1:34332 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,271 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0005 with negotiated timeout 10000 for client /127.0.0.1:34332 -hbase_1 | 2020-07-22 09:39:32,271 INFO [hbase:41229.activeMasterManager-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x17375e441fa0005, negotiated timeout = 10000 -hbase_1 | hbase stderr | 2020-07-22 09:39:32,276 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0005 type:create cxid:0x1 zxid:0x21023 txntype:-1 reqpath:n/a Error Path:/hbase/replication/rs Error:KeeperErrorCode = NodeExists for /hbase/replication/rs -hbase_1 | hbase stderr | 2020-07-22 09:39:32,284 INFO [hbase:41229.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 0, slept for 0 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms. -hbase_1 | hbase stderr | 2020-07-22 09:39:32,288 WARN [RS:0;hbase:41633] regionserver.HRegionServer: reportForDuty failed; sleeping and then retrying. -hbase_1 | hbase stderr | 2020-07-22 09:39:33,787 INFO [hbase:41229.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 0, slept for 1503 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms. -hbase_1 | hbase stderr | 2020-07-22 09:39:35,289 INFO [RS:0;hbase:41633] regionserver.HRegionServer: reportForDuty to master=hbase,41229,1595410761736 with port=41633, startcode=1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:35,290 INFO [hbase:41229.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 0, slept for 3006 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms. -hbase_1 | hbase stderr | 2020-07-22 09:39:35,302 INFO [B.defaultRpcServer.handler=5,queue=2,port=41229] master.ServerManager: Registering server=hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:35,312 WARN [RS:0;hbase:41633] hbase.ZNodeClearer: Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) -hbase_1 | hbase stderr | 2020-07-22 09:39:35,315 INFO [RS:0;hbase:41633] hfile.CacheConfig: Allocating LruBlockCache size=3.10 GB, blockSize=64 KB -hbase_1 | hbase stderr | 2020-07-22 09:39:35,324 INFO [RS:0;hbase:41633] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=0, currentSize=3412528, freeSize=3323299792, maxSize=3326712320, heapSize=3412528, minSize=3160376576, minFactor=0.95, multiSize=1580188288, multiFactor=0.5, singleSize=790094144, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false -hbase_1 | hbase stderr | 2020-07-22 09:39:35,340 INFO [hbase:41229.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 1, slept for 3056 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms. -hbase_1 | hbase stderr | 2020-07-22 09:39:35,372 INFO [RS:0;hbase:41633] wal.WALFactory: Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.DefaultWALProvider -hbase_1 | hbase stderr | 2020-07-22 09:39:35,385 INFO [RS:0;hbase:41633] wal.FSHLog: WAL configuration: blocksize=32 MB, rollsize=30.40 MB, prefix=hbase%2C41633%2C1595410763148.default, suffix=, logDir=file:/data/hbase/WALs/hbase,41633,1595410763148, archiveDir=file:/data/hbase/oldWALs -hbase_1 | hbase stderr | 2020-07-22 09:39:35,423 INFO [RS:0;hbase:41633] wal.FSHLog: Slow sync cost: 0 ms, current pipeline: [] -hbase_1 | hbase stderr | 2020-07-22 09:39:35,424 INFO [RS:0;hbase:41633] wal.FSHLog: New WAL /data/hbase/WALs/hbase,41633,1595410763148/hbase%2C41633%2C1595410763148.default.1595410775385 -hbase_1 | hbase stderr | 2020-07-22 09:39:35,450 INFO [RS:0;hbase:41633] regionserver.MetricsRegionServerWrapperImpl: Computing regionserver metrics every 5000 milliseconds -hbase_1 | hbase stderr | 2020-07-22 09:39:35,458 INFO [RS:0;hbase:41633] regionserver.ReplicationSourceManager: Current list of replicators: [hbase,37695,1595404637854, hbase,41633,1595410763148, hbase,43049,1595410682202] other RSs: [hbase,41633,1595410763148] -hbase_1 | hbase stderr | 2020-07-22 09:39:35,498 INFO [SplitLogWorker-hbase:41633] regionserver.SplitLogWorker: SplitLogWorker hbase,41633,1595410763148 starting -hbase_1 | 2020-07-22 09:39:35,500 INFO [RS:0;hbase:41633] regionserver.HeapMemoryManager: Starting HeapMemoryTuner chore. -hbase_1 | hbase stderr | 2020-07-22 09:39:35,502 INFO [RS:0;hbase:41633] regionserver.HRegionServer: Serving as hbase,41633,1595410763148, RpcServer on hbase/172.18.0.2:41633, sessionid=0x17375e441fa0001 -hbase_1 | hbase stderr | 2020-07-22 09:39:35,514 INFO [RS:0;hbase:41633] quotas.RegionServerQuotaManager: Quota support disabled -hbase_1 | hbase stderr | 2020-07-22 09:39:35,534 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53314 -hbase_1 | hbase stderr | 2020-07-22 09:39:35,537 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53314 -hbase_1 | hbase stderr | 2020-07-22 09:39:35,539 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0006 with negotiated timeout 40000 for client /172.18.0.4:53314 -hbase_1 | hbase stderr | 2020-07-22 09:39:36,845 INFO [hbase:41229.activeMasterManager] master.ServerManager: Finished waiting for region servers count to settle; checked in 1, slept for 4561 ms, expecting minimum of 1, maximum of 2147483647, master is running -hbase_1 | hbase stderr | 2020-07-22 09:39:36,846 INFO [hbase:41229.activeMasterManager] master.MasterFileSystem: Log folder file:/data/hbase/WALs/hbase,43049,1595410682202 doesn't belong to a known region server, splitting -hbase_1 | hbase stderr | 2020-07-22 09:39:36,847 INFO [hbase:41229.activeMasterManager] master.MasterFileSystem: Log folder file:/data/hbase/WALs/hbase,41633,1595410763148 belongs to an existing region server -hbase_1 | hbase stderr | 2020-07-22 09:39:36,850 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: dead splitlog workers [hbase,43049,1595410682202] -hbase_1 | hbase stderr | 2020-07-22 09:39:36,851 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: Started splitting 1 logs in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] for [hbase,43049,1595410682202] -hbase_1 | hbase stderr | 2020-07-22 09:39:36,861 INFO [SplitLogWorker-hbase:41633] coordination.ZkSplitLogWorkerCoordination: worker hbase,41633,1595410763148 acquired task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta -hbase_1 | hbase stderr | 2020-07-22 09:39:36,861 INFO [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta acquired by hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:36,885 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] wal.WALSplitter: Splitting wal: file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202..meta.1595410688276.meta, length=1016 -hbase_1 | 2020-07-22 09:39:36,886 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] wal.WALSplitter: DistributedLogReplay = false -hbase_1 | hbase stderr | 2020-07-22 09:39:36,939 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] wal.WALSplitter: 3 split writers finished; closing... -hbase_1 | hbase stderr | 2020-07-22 09:39:36,943 INFO [split-log-closeStream-1] wal.WALSplitter: Rename file:/data/hbase/data/hbase/meta/1588230740/recovered.edits/0000000000000000677.temp to file:/data/hbase/data/hbase/meta/1588230740/recovered.edits/0000000000000000678 -hbase_1 | hbase stderr | 2020-07-22 09:39:36,944 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] wal.WALSplitter: Processed 2 edits across 1 regions; edits skipped=1; log file=file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202..meta.1595410688276.meta, length=1016, corrupted=false, progress failed=false -hbase_1 | hbase stderr | 2020-07-22 09:39:36,947 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] coordination.ZkSplitLogWorkerCoordination: successfully transitioned task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta to final state DONE hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:36,947 INFO [RS_LOG_REPLAY_OPS-hbase:41633-0] handler.WALSplitterHandler: worker hbase,41633,1595410763148 done with task org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination$ZkSplitTaskDetails@bb64aec in 84ms -hbase_1 | 2020-07-22 09:39:36,947 INFO [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta entered state: DONE hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:36,948 INFO [main-EventThread] wal.WALSplitter: Archived processed log file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202..meta.1595410688276.meta to file:/data/hbase/oldWALs/hbase%2C43049%2C1595410682202..meta.1595410688276.meta -hbase_1 | hbase stderr | 2020-07-22 09:39:36,949 INFO [main-EventThread] coordination.SplitLogManagerCoordination: Done splitting /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202..meta.1595410688276.meta -hbase_1 | hbase stderr | 2020-07-22 09:39:36,950 WARN [hbase:41229.activeMasterManager] master.SplitLogManager: Returning success without actually splitting and deleting all the log files in path file:/data/hbase/WALs/hbase,43049,1595410682202-splitting: [DeprecatedRawLocalFileStatus{path=file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976; isDirectory=false; length=669; replication=1; blocksize=33554432; modification_time=1595410689000; access_time=0; owner=; group=; permission=rw-rw-rw-; isSymlink=false}] -hbase_1 | java.io.IOException: Directory /data/hbase/WALs/hbase,43049,1595410682202-splitting is not empty -hbase_1 | at org.apache.hadoop.fs.RawLocalFileSystem.delete(RawLocalFileSystem.java:360) -hbase_1 | at org.apache.hadoop.fs.ChecksumFileSystem.delete(ChecksumFileSystem.java:540) -hbase_1 | at org.apache.hadoop.hbase.master.SplitLogManager.splitLogDistributed(SplitLogManager.java:296) -hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitLog(MasterFileSystem.java:398) -hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitMetaLog(MasterFileSystem.java:313) -hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitMetaLog(MasterFileSystem.java:304) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster.splitMetaLogBeforeAssignment(HMaster.java:989) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:695) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster.access$500(HMaster.java:184) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster$1.run(HMaster.java:1729) -hbase_1 | at java.lang.Thread.run(Thread.java:745) -hbase_1 | hbase stderr | 2020-07-22 09:39:36,951 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: finished splitting (more than or equal to) 1016 bytes in 1 log files in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] in 100ms -hbase_1 | hbase stderr | 2020-07-22 09:39:36,964 INFO [hbase:41229.activeMasterManager] zookeeper.MetaTableLocator: Failed verification of hbase:meta,,1 at address=hbase,43049,1595410682202, exception=Connection refused -hbase_1 | hbase stderr | 2020-07-22 09:39:36,965 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: dead splitlog workers [hbase,43049,1595410682202] -hbase_1 | hbase stderr | 2020-07-22 09:39:36,965 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: file:/data/hbase/WALs/hbase,43049,1595410682202-splitting is empty dir, no logs to split -hbase_1 | hbase stderr | 2020-07-22 09:39:36,965 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: Started splitting 0 logs in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] for [hbase,43049,1595410682202] -hbase_1 | hbase stderr | 2020-07-22 09:39:36,965 WARN [hbase:41229.activeMasterManager] master.SplitLogManager: Returning success without actually splitting and deleting all the log files in path file:/data/hbase/WALs/hbase,43049,1595410682202-splitting: [DeprecatedRawLocalFileStatus{path=file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976; isDirectory=false; length=669; replication=1; blocksize=33554432; modification_time=1595410689000; access_time=0; owner=; group=; permission=rw-rw-rw-; isSymlink=false}] -hbase_1 | java.io.IOException: Directory /data/hbase/WALs/hbase,43049,1595410682202-splitting is not empty -hbase_1 | at org.apache.hadoop.fs.RawLocalFileSystem.delete(RawLocalFileSystem.java:360) -hbase_1 | at org.apache.hadoop.fs.ChecksumFileSystem.delete(ChecksumFileSystem.java:540) -hbase_1 | at org.apache.hadoop.hbase.master.SplitLogManager.splitLogDistributed(SplitLogManager.java:296) -hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitLog(MasterFileSystem.java:398) -hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitMetaLog(MasterFileSystem.java:313) -hbase_1 | at org.apache.hadoop.hbase.master.MasterFileSystem.splitMetaLog(MasterFileSystem.java:304) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster.splitMetaLogBeforeAssignment(HMaster.java:989) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster.assignMeta(HMaster.java:917) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:728) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster.access$500(HMaster.java:184) -hbase_1 | at org.apache.hadoop.hbase.master.HMaster$1.run(HMaster.java:1729) -hbase_1 | at java.lang.Thread.run(Thread.java:745) -hbase_1 | hbase stderr | 2020-07-22 09:39:36,966 INFO [hbase:41229.activeMasterManager] master.SplitLogManager: finished splitting (more than or equal to) 0 bytes in 0 log files in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] in 1ms -hbase_1 | 2020-07-22 09:39:36,966 INFO [hbase:41229.activeMasterManager] zookeeper.MetaTableLocator: Deleting hbase:meta region location in ZooKeeper -hbase_1 | hbase stderr | 2020-07-22 09:39:36,969 INFO [hbase:41229.activeMasterManager] master.AssignmentManager: Setting node as OFFLINED in ZooKeeper for region {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} -hbase_1 | hbase stderr | 2020-07-22 09:39:36,975 INFO [hbase:41229.activeMasterManager] master.AssignmentManager: Assigning hbase:meta,,1.1588230740 to hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:36,975 INFO [hbase:41229.activeMasterManager] master.RegionStates: Transition {1588230740 state=OFFLINE, ts=1595410776969, server=null} to {1588230740 state=PENDING_OPEN, ts=1595410776975, server=hbase,41633,1595410763148} -hbase_1 | hbase stderr | 2020-07-22 09:39:36,985 INFO [PriorityRpcServer.handler=0,queue=0,port=41633] regionserver.RSRpcServices: Open hbase:meta,,1.1588230740 -hbase_1 | hbase stderr | 2020-07-22 09:39:36,990 INFO [hbase:41229.activeMasterManager] master.ServerManager: AssignmentManager hasn't finished failover cleanup; waiting -hbase_1 | hbase stderr | 2020-07-22 09:39:36,994 INFO [RS_OPEN_META-hbase:41633-0] wal.WALFactory: Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.DefaultWALProvider -hbase_1 | hbase stderr | 2020-07-22 09:39:36,995 INFO [RS_OPEN_META-hbase:41633-0] wal.FSHLog: WAL configuration: blocksize=32 MB, rollsize=30.40 MB, prefix=hbase%2C41633%2C1595410763148..meta, suffix=.meta, logDir=file:/data/hbase/WALs/hbase,41633,1595410763148, archiveDir=file:/data/hbase/oldWALs -hbase_1 | hbase stderr | 2020-07-22 09:39:36,996 INFO [AM.ZK.Worker-pool2-t1] master.RegionStates: Transition {1588230740 state=PENDING_OPEN, ts=1595410776975, server=hbase,41633,1595410763148} to {1588230740 state=OPENING, ts=1595410776996, server=hbase,41633,1595410763148} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,006 INFO [RS_OPEN_META-hbase:41633-0] wal.FSHLog: Slow sync cost: 0 ms, current pipeline: [] -hbase_1 | hbase stderr | 2020-07-22 09:39:37,006 INFO [RS_OPEN_META-hbase:41633-0] wal.FSHLog: New WAL /data/hbase/WALs/hbase,41633,1595410763148/hbase%2C41633%2C1595410763148..meta.1595410776995.meta -hbase_1 | hbase stderr | 2020-07-22 09:39:37,041 INFO [RS_OPEN_META-hbase:41633-0] regionserver.RegionCoprocessorHost: Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. -hbase_1 | hbase stderr | 2020-07-22 09:39:37,084 INFO [StoreOpener-1588230740-1] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=0, currentSize=3412528, freeSize=3323299792, maxSize=3326712320, heapSize=3412528, minSize=3160376576, minFactor=0.95, multiSize=1580188288, multiFactor=0.5, singleSize=790094144, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false -hbase_1 | hbase stderr | 2020-07-22 09:39:37,090 INFO [StoreOpener-1588230740-1] compactions.CompactionConfiguration: size [134217728, 9223372036854775807, 9223372036854775807); files [3, 10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,136 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HRegion: Replaying edits from file:/data/hbase/data/hbase/meta/1588230740/recovered.edits/0000000000000000678 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,141 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HRegion: Flushing 1/1 column families, memstore=1.38 KB; WAL is null, using passed sequenceid=678 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,189 INFO [RS_OPEN_META-hbase:41633-0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=678, memsize=1.4 K, hasBloomFilter=false, into tmp file file:/data/hbase/data/hbase/meta/1588230740/.tmp/ec2b6cf702f1488f8d734d1e8a2a5d6a -hbase_1 | hbase stderr | 2020-07-22 09:39:37,210 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HStore: Added file:/data/hbase/data/hbase/meta/1588230740/info/ec2b6cf702f1488f8d734d1e8a2a5d6a, entries=6, sequenceid=678, filesize=5.3 K -hbase_1 | hbase stderr | 2020-07-22 09:39:37,211 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HRegion: Finished memstore flush of ~1.38 KB/1416, currentsize=0 B/0 for region hbase:meta,,1.1588230740 in 71ms, sequenceid=678, compaction requested=false; wal=null -hbase_1 | hbase stderr | 2020-07-22 09:39:37,223 INFO [RS_OPEN_META-hbase:41633-0] regionserver.HRegion: Onlined 1588230740; next sequenceid=679 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,234 INFO [PostOpenDeployTasks:1588230740] regionserver.HRegionServer: Post open deploy tasks for hbase:meta,,1.1588230740 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,235 INFO [PostOpenDeployTasks:1588230740] zookeeper.MetaTableLocator: Setting hbase:meta region location in ZooKeeper as hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,236 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0001 type:setData cxid:0x39 zxid:0x2102e txntype:-1 reqpath:n/a Error Path:/hbase/meta-region-server Error:KeeperErrorCode = NoNode for /hbase/meta-region-server -hbase_1 | hbase stderr | 2020-07-22 09:39:37,241 INFO [AM.ZK.Worker-pool2-t2] master.RegionStates: Transition {1588230740 state=OPENING, ts=1595410776996, server=hbase,41633,1595410763148} to {1588230740 state=OPEN, ts=1595410777241, server=hbase,41633,1595410763148} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,242 INFO [AM.ZK.Worker-pool2-t2] coordination.ZkOpenRegionCoordination: Handling OPENED of 1588230740 from hbase,41229,1595410761736; deleting unassigned node -hbase_1 | hbase stderr | 2020-07-22 09:39:37,249 INFO [hbase:41229.activeMasterManager] master.HMaster: hbase:meta with replicaId 0 assigned=1, rit=false, location=hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,309 INFO [hbase:41229.activeMasterManager] hbase.MetaMigrationConvertingToPB: META already up-to date with PB serialization -hbase_1 | hbase stderr | 2020-07-22 09:39:37,322 INFO [hbase:41229.activeMasterManager] master.RegionStates: Transition {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OPEN, ts=1595410777322, server=hbase,43049,1595410682202} to {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OFFLINE, ts=1595410777322, server=hbase,43049,1595410682202} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,322 INFO [hbase:41229.activeMasterManager] master.RegionStates: Transition {d0d5370755b75937494d142505e5a2a4 state=OPEN, ts=1595410777322, server=hbase,43049,1595410682202} to {d0d5370755b75937494d142505e5a2a4 state=OFFLINE, ts=1595410777322, server=hbase,43049,1595410682202} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,323 INFO [hbase:41229.activeMasterManager] master.AssignmentManager: Found regions out on cluster or in RIT; presuming failover -hbase_1 | hbase stderr | 2020-07-22 09:39:37,442 INFO [hbase:41229.activeMasterManager] master.AssignmentManager: Joined the cluster in 133ms, failover=true -hbase_1 | hbase stderr | 2020-07-22 09:39:37,443 INFO [ProcedureExecutor-0] procedure.ServerCrashProcedure: Start processing crashed hbase,43049,1595410682202 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,650 INFO [ProcedureExecutor-2] master.SplitLogManager: dead splitlog workers [hbase,43049,1595410682202] -hbase_1 | hbase stderr | 2020-07-22 09:39:37,651 INFO [ProcedureExecutor-2] master.SplitLogManager: Started splitting 1 logs in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] for [hbase,43049,1595410682202] -hbase_1 | hbase stderr | 2020-07-22 09:39:37,655 INFO [SplitLogWorker-hbase:41633] coordination.ZkSplitLogWorkerCoordination: worker hbase,41633,1595410763148 acquired task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,655 INFO [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 acquired by hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,669 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] wal.WALSplitter: Splitting wal: file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976, length=669 -hbase_1 | 2020-07-22 09:39:37,669 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] wal.WALSplitter: DistributedLogReplay = false -hbase_1 | hbase stderr | 2020-07-22 09:39:37,686 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] wal.WALSplitter: 3 split writers finished; closing... -hbase_1 | hbase stderr | 2020-07-22 09:39:37,686 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] wal.WALSplitter: Processed 0 edits across 0 regions; edits skipped=2; log file=file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976, length=669, corrupted=false, progress failed=false -hbase_1 | hbase stderr | 2020-07-22 09:39:37,687 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] coordination.ZkSplitLogWorkerCoordination: successfully transitioned task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 to final state DONE hbase,41633,1595410763148 -foxtrot_server | INFO [2020-07-22 09:39:37,696] [main] [HBaseUtil]: Table foxtrot already exists. Nothing to do. -foxtrot_server | INFO [2020-07-22 09:39:37,709] [main] [InitializerCommand]: Initialization complete... -foxtrot_server | Starting foxtrot with command line: java -Dfile.encoding=utf-8 -XX:+UseG1GC -Xms1g -Xmx1g -jar server.jar server /config/docker.yml -hbase_1 | hbase stderr | 2020-07-22 09:39:37,687 INFO [RS_LOG_REPLAY_OPS-hbase:41633-1] handler.WALSplitterHandler: worker hbase,41633,1595410763148 done with task org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination$ZkSplitTaskDetails@7dfa1b92 in 32ms -hbase_1 | hbase stderr | 2020-07-22 09:39:37,687 INFO [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 entered state: DONE hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,688 INFO [main-EventThread] wal.WALSplitter: Archived processed log file:/data/hbase/WALs/hbase,43049,1595410682202-splitting/hbase%2C43049%2C1595410682202.default.1595410683976 to file:/data/hbase/oldWALs/hbase%2C43049%2C1595410682202.default.1595410683976 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,688 INFO [main-EventThread] coordination.SplitLogManagerCoordination: Done splitting /hbase/splitWAL/WALs%2Fhbase%2C43049%2C1595410682202-splitting%2Fhbase%252C43049%252C1595410682202.default.1595410683976 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,689 INFO [ProcedureExecutor-2] master.SplitLogManager: finished splitting (more than or equal to) 669 bytes in 1 log files in [file:/data/hbase/WALs/hbase,43049,1595410682202-splitting] in 38ms -hbase_1 | hbase stderr | 2020-07-22 09:39:37,697 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e441fa0006 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,699 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxn: Closed socket connection for client /172.18.0.4:53314 which had sessionid 0x17375e441fa0006 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,791 INFO [ProcedureExecutor-2] master.AssignmentManager: Assigning 2 region(s) to hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,798 INFO [ProcedureExecutor-2] master.RegionStates: Transition {d0d5370755b75937494d142505e5a2a4 state=OFFLINE, ts=1595410777793, server=hbase,43049,1595410682202} to {d0d5370755b75937494d142505e5a2a4 state=PENDING_OPEN, ts=1595410777798, server=hbase,41633,1595410763148} -hbase_1 | 2020-07-22 09:39:37,798 INFO [ProcedureExecutor-2] master.RegionStates: Transition {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OFFLINE, ts=1595410777793, server=hbase,43049,1595410682202} to {d7c0a1ab1664e7d809c8b3dcacccddf7 state=PENDING_OPEN, ts=1595410777798, server=hbase,41633,1595410763148} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,801 INFO [PriorityRpcServer.handler=2,queue=0,port=41633] regionserver.RSRpcServices: Open hbase:namespace,,1586108027618.d0d5370755b75937494d142505e5a2a4. -hbase_1 | hbase stderr | 2020-07-22 09:39:37,802 INFO [PriorityRpcServer.handler=2,queue=0,port=41633] regionserver.RSRpcServices: Open foxtrot,,1586108035639.d7c0a1ab1664e7d809c8b3dcacccddf7. -hbase_1 | hbase stderr | 2020-07-22 09:39:37,807 INFO [AM.ZK.Worker-pool2-t5] master.RegionStates: Transition {d0d5370755b75937494d142505e5a2a4 state=PENDING_OPEN, ts=1595410777798, server=hbase,41633,1595410763148} to {d0d5370755b75937494d142505e5a2a4 state=OPENING, ts=1595410777807, server=hbase,41633,1595410763148} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,811 INFO [AM.ZK.Worker-pool2-t6] master.RegionStates: Transition {d7c0a1ab1664e7d809c8b3dcacccddf7 state=PENDING_OPEN, ts=1595410777798, server=hbase,41633,1595410763148} to {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OPENING, ts=1595410777811, server=hbase,41633,1595410763148} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,814 INFO [RS_OPEN_REGION-hbase:41633-1] compress.CodecPool: Got brand-new compressor [.gz] -hbase_1 | hbase stderr | 2020-07-22 09:39:37,826 INFO [StoreOpener-d0d5370755b75937494d142505e5a2a4-1] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=2, currentSize=3421240, freeSize=3323291080, maxSize=3326712320, heapSize=3421240, minSize=3160376576, minFactor=0.95, multiSize=1580188288, multiFactor=0.5, singleSize=790094144, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false -hbase_1 | hbase stderr | 2020-07-22 09:39:37,826 INFO [StoreOpener-d0d5370755b75937494d142505e5a2a4-1] compactions.CompactionConfiguration: size [134217728, 9223372036854775807, 9223372036854775807); files [3, 10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,838 INFO [StoreOpener-d7c0a1ab1664e7d809c8b3dcacccddf7-1] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=2, currentSize=3421240, freeSize=3323291080, maxSize=3326712320, heapSize=3421240, minSize=3160376576, minFactor=0.95, multiSize=1580188288, multiFactor=0.5, singleSize=790094144, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false -hbase_1 | hbase stderr | 2020-07-22 09:39:37,839 INFO [StoreOpener-d7c0a1ab1664e7d809c8b3dcacccddf7-1] compactions.CompactionConfiguration: size [134217728, 9223372036854775807, 9223372036854775807); files [3, 10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,847 INFO [StoreFileOpenerThread-d-1] compress.CodecPool: Got brand-new decompressor [.gz] -hbase_1 | hbase stderr | 2020-07-22 09:39:37,850 INFO [StoreFileOpenerThread-d-1] compress.CodecPool: Got brand-new decompressor [.gz] -hbase_1 | hbase stderr | 2020-07-22 09:39:37,850 INFO [StoreFileOpenerThread-d-1] compress.CodecPool: Got brand-new decompressor [.gz] -hbase_1 | hbase stderr | 2020-07-22 09:39:37,850 INFO [StoreFileOpenerThread-d-1] compress.CodecPool: Got brand-new decompressor [.gz] -hbase_1 | hbase stderr | 2020-07-22 09:39:37,871 INFO [RS_OPEN_REGION-hbase:41633-1] regionserver.HRegion: Onlined d7c0a1ab1664e7d809c8b3dcacccddf7; next sequenceid=10236 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,873 INFO [RS_OPEN_REGION-hbase:41633-0] regionserver.HRegion: Onlined d0d5370755b75937494d142505e5a2a4; next sequenceid=195 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,882 INFO [PostOpenDeployTasks:d7c0a1ab1664e7d809c8b3dcacccddf7] regionserver.HRegionServer: Post open deploy tasks for foxtrot,,1586108035639.d7c0a1ab1664e7d809c8b3dcacccddf7. -hbase_1 | hbase stderr | 2020-07-22 09:39:37,892 INFO [PostOpenDeployTasks:d0d5370755b75937494d142505e5a2a4] regionserver.HRegionServer: Post open deploy tasks for hbase:namespace,,1586108027618.d0d5370755b75937494d142505e5a2a4. -hbase_1 | hbase stderr | 2020-07-22 09:39:37,961 INFO [PostOpenDeployTasks:d7c0a1ab1664e7d809c8b3dcacccddf7] hbase.MetaTableAccessor: Updated row foxtrot,,1586108035639.d7c0a1ab1664e7d809c8b3dcacccddf7. with server=hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,961 INFO [PostOpenDeployTasks:d0d5370755b75937494d142505e5a2a4] hbase.MetaTableAccessor: Updated row hbase:namespace,,1586108027618.d0d5370755b75937494d142505e5a2a4. with server=hbase,41633,1595410763148 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,975 INFO [AM.ZK.Worker-pool2-t8] master.RegionStates: Transition {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OPENING, ts=1595410777811, server=hbase,41633,1595410763148} to {d7c0a1ab1664e7d809c8b3dcacccddf7 state=OPEN, ts=1595410777975, server=hbase,41633,1595410763148} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,976 INFO [AM.ZK.Worker-pool2-t7] master.RegionStates: Transition {d0d5370755b75937494d142505e5a2a4 state=OPENING, ts=1595410777807, server=hbase,41633,1595410763148} to {d0d5370755b75937494d142505e5a2a4 state=OPEN, ts=1595410777976, server=hbase,41633,1595410763148} -hbase_1 | hbase stderr | 2020-07-22 09:39:37,979 INFO [AM.ZK.Worker-pool2-t10] master.RegionStates: Offlined d7c0a1ab1664e7d809c8b3dcacccddf7 from hbase,43049,1595410682202 -hbase_1 | hbase stderr | 2020-07-22 09:39:37,980 INFO [AM.ZK.Worker-pool2-t11] master.RegionStates: Offlined d0d5370755b75937494d142505e5a2a4 from hbase,43049,1595410682202 -hbase_1 | hbase stderr | 2020-07-22 09:39:38,082 INFO [ProcedureExecutor-2] procedure.ServerCrashProcedure: Finished processing of crashed hbase,43049,1595410682202 -foxtrot_server | ERROR StatusLogger Log4j2 could not find a logging implementation. Please add log4j-core to the classpath. Using SimpleLogger to log to the console... -foxtrot_server | INFO [2020-07-22 09:39:39,444] [main] [DefaultServerFactory]: Registering jersey handler with root path prefix: / -foxtrot_server | INFO [2020-07-22 09:39:39,447] [main] [DefaultServerFactory]: Registering admin handler with root path prefix: / -foxtrot_server | INFO [2020-07-22 09:39:39,447] [main] [AssetsBundle]: Registering AssetBundle with name: console for path /* -foxtrot_server | INFO [2020-07-22 09:39:39,474] [main] [AssetsBundle]: Registering AssetBundle with name: swagger-assets for path /foxtrot/swagger-static/* -foxtrot_server | INFO [2020-07-22 09:39:39,474] [main] [AssetsBundle]: Registering AssetBundle with name: swagger-oauth2-connect for path /foxtrot/oauth2-redirect.html/* -foxtrot_server | INFO [2020-07-22 09:39:43,919] [main] [Reflections]: Reflections took 4424 ms to scan 1 urls, producing 79344 keys and 149875 values -foxtrot_server | WARN [2020-07-22 09:39:44,666] [main] [ExecutorServiceBuilder]: Parameter 'maximumPoolSize' is conflicting with unbounded work queues -foxtrot_server | INFO [2020-07-22 09:39:44,690] [main] [ManagedInstaller]: managed = -foxtrot_server | -foxtrot_server | (com.flipkart.foxtrot.core.datastore.impl.hbase.HbaseTableConnection) -foxtrot_server | (com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection) -foxtrot_server | (com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection) -foxtrot_server | (com.flipkart.foxtrot.core.table.impl.DistributedTableMetadataManager) -foxtrot_server | (com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader) -foxtrot_server | (com.flipkart.foxtrot.core.common.DataDeletionManager) -foxtrot_server | (com.flipkart.foxtrot.server.cluster.ClusterManager) -foxtrot_server | (com.flipkart.foxtrot.core.cardinality.CardinalityCalculationManager) -foxtrot_server | (com.flipkart.foxtrot.core.internalevents.EventBusInitializer) -foxtrot_server | (com.flipkart.foxtrot.core.jobs.optimization.EsIndexOptimizationManager) -foxtrot_server | (com.flipkart.foxtrot.server.jobs.consolehistory.ConsoleHistoryManager) -foxtrot_server | (com.flipkart.foxtrot.server.auth.sessionstore.DistributedSessionDataStore) -foxtrot_server | (com.flipkart.foxtrot.server.jobs.sessioncleanup.ExpiredSessionsCleaner) -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:44,691] [main] [HealthCheckInstaller]: health checks = -foxtrot_server | -foxtrot_server | hazelcastHealthcheck (com.flipkart.foxtrot.server.healthcheck.HazelcastHealthCheck) -foxtrot_server | hbaseHealthcheck (com.flipkart.foxtrot.server.healthcheck.HBaseHealthCheck) -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:44,692] [main] [WebFilterInstaller]: filters = -foxtrot_server | -foxtrot_server | /* (com.flipkart.foxtrot.server.auth.filter.UserAuthenticationFilter) .userauthentication -foxtrot_server | -foxtrot_server | WARN [2020-07-22 09:39:44,725] [main] [NativeCodeLoader]: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -foxtrot_server | INFO [2020-07-22 09:39:45,426] [main] [ServerFactory]: Starting foxtrot -foxtrot_server | _______ ______ ___ ___ .___________..______ ______ .___________. -foxtrot_server | | ____| / __ \ \ \ / / | || _ \ / __ \ | | -foxtrot_server | | |__ | | | | \ V / `---| |----`| |_) | | | | | `---| |----` -foxtrot_server | | __| | | | | > < | | | / | | | | | | -foxtrot_server | | | | `--' | / . \ | | | |\ \----.| `--' | | | -foxtrot_server | |__| \______/ /__/ \__\ |__| | _| `._____| \______/ |__| -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:45,532] [main] [SetUIDListener]: Opened application@7b2c82c{HTTP/1.1,[http/1.1]}{0.0.0.0:17000} -foxtrot_server | INFO [2020-07-22 09:39:45,532] [main] [SetUIDListener]: Opened admin@30fa6986{HTTP/1.1,[http/1.1]}{0.0.0.0:17001} -foxtrot_server | INFO [2020-07-22 09:39:45,535] [main] [Server]: jetty-9.4.z-SNAPSHOT; built: 2019-04-29T20:42:08.989Z; git: e1bc35120a6617ee3df052294e433f3a25ce7097; jvm 1.8.0_252-8u252-b09-1~16.04-b09 -foxtrot_server | INFO [2020-07-22 09:39:45,549] [main] [HbaseTableConnection]: Starting HBase Connection -foxtrot_server | INFO [2020-07-22 09:39:45,641] [main] [HbaseTableConnection]: Started HBase Connection -foxtrot_server | INFO [2020-07-22 09:39:45,641] [main] [ElasticsearchConnection]: Starting ElasticSearch Client -foxtrot_server | INFO [2020-07-22 09:39:46,241] [main] [ElasticsearchConnection]: Started ElasticSearch Client -foxtrot_server | INFO [2020-07-22 09:39:46,242] [main] [HazelcastConnection]: Starting Hazelcast Instance -foxtrot_server | WARN [2020-07-22 09:39:46,342] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] You configured your member address as host name. Please be aware of that your dns can be spoofed. Make sure that your dns configurations are correct. -foxtrot_server | INFO [2020-07-22 09:39:46,342] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] Resolving domain name 'fac2f46c5b7f' to address(es): [172.18.0.4] -foxtrot_server | INFO [2020-07-22 09:39:46,342] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] Interfaces is disabled, trying to pick one address from TCP-IP config addresses: [fac2f46c5b7f/172.18.0.4] -foxtrot_server | INFO [2020-07-22 09:39:46,342] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] Prefer IPv4 stack is true, prefer IPv6 addresses is false -foxtrot_server | INFO [2020-07-22 09:39:46,345] [main] [AddressPicker]: [LOCAL] [foxtrot] [4.0.2] Picked [fac2f46c5b7f]:5701, using socket ServerSocket[addr=/0.0.0.0,localport=5701], bind any local is true -foxtrot_server | INFO [2020-07-22 09:39:46,375] [main] [system]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Hazelcast 4.0.2 (20200702 - 2de3027) starting at [fac2f46c5b7f]:5701 -foxtrot_server | INFO [2020-07-22 09:39:46,375] [main] [system]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Copyright (c) 2008-2020, Hazelcast, Inc. All Rights Reserved. -foxtrot_server | INFO [2020-07-22 09:39:46,694] [main] [BackpressureRegulator]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Backpressure is disabled -foxtrot_server | INFO [2020-07-22 09:39:47,116] [main] [Node]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Creating TcpIpJoiner -foxtrot_server | WARN [2020-07-22 09:39:47,118] [main] [CPSubsystem]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] CP Subsystem is not enabled. CP data structures will operate in UNSAFE mode! Please note that UNSAFE mode will not provide strong consistency guarantees. -foxtrot_server | INFO [2020-07-22 09:39:47,423] [main] [OperationExecutorImpl]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Starting 8 partition threads and 5 generic threads (1 dedicated for priority tasks) -foxtrot_server | INFO [2020-07-22 09:39:47,432] [main] [Diagnostics]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Diagnostics disabled. To enable add -Dhazelcast.diagnostics.enabled=true to the JVM arguments. -foxtrot_server | INFO [2020-07-22 09:39:47,446] [main] [LifecycleService]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] [fac2f46c5b7f]:5701 is STARTING -foxtrot_server | INFO [2020-07-22 09:39:47,480] [main] [ClusterService]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] -foxtrot_server | -foxtrot_server | Members {size:1, ver:1} [ -foxtrot_server | Member [fac2f46c5b7f]:5701 - 259b100b-6b86-4551-a0cf-775a4676c26d this -foxtrot_server | ] -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:47,495] [main] [LifecycleService]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] [fac2f46c5b7f]:5701 is STARTED -foxtrot_server | INFO [2020-07-22 09:39:47,495] [main] [HazelcastConnection]: Started Hazelcast Instance -foxtrot_server | INFO [2020-07-22 09:39:47,526] [main] [PartitionStateManager]: [fac2f46c5b7f]:5701 [foxtrot] [4.0.2] Initializing cluster partition table arrangement... -foxtrot_server | INFO [2020-07-22 09:39:47,613] [hz.foxtrot-fac2f46c5b7f-1595410784519.cached.thread-3] [TableMapStore]: Load all keys called -foxtrot_server | INFO [2020-07-22 09:39:47,927] [hz.foxtrot-fac2f46c5b7f-1595410784519.cached.thread-3] [TableMapStore]: Loaded value count: 0 -foxtrot_server | INFO [2020-07-22 09:39:48,446] [main] [Reflections]: Reflections took 130 ms to scan 1 urls, producing 86 keys and 237 values -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.TrendAction -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.FilterAction -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.MultiTimeQueryAction -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.DistinctAction -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.MultiQueryAction -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.HistogramAction -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.StatsAction -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.CountAction -foxtrot_server | INFO [2020-07-22 09:39:48,447] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.StatsTrendAction -foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [AnalyticsLoader]: Registered action: com.flipkart.foxtrot.core.querystore.actions.GroupAction -foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [DataDeletionManager]: Starting Deletion Manager -foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [DataDeletionManager]: Scheduling data deletion Job -foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [DataDeletionManager]: Scheduled data deletion Job -foxtrot_server | INFO [2020-07-22 09:39:48,448] [main] [DataDeletionManager]: Started Deletion Manager -foxtrot_server | INFO [2020-07-22 09:39:48,449] [main] [BaseJobManager]: Starting CardinalityConfigJob Manager -foxtrot_server | INFO [2020-07-22 09:39:48,449] [main] [BaseJobManager]: Scheduling CardinalityConfigJob Job -foxtrot_server | INFO [2020-07-22 09:39:48,452] [main] [BaseJobManager]: Scheduled CardinalityConfigJob Job -foxtrot_server | INFO [2020-07-22 09:39:48,454] [main] [BaseJobManager]: Starting ESIndexOptimizer Manager -foxtrot_server | INFO [2020-07-22 09:39:48,454] [main] [BaseJobManager]: Scheduling ESIndexOptimizer Job -foxtrot_server | INFO [2020-07-22 09:39:48,454] [main] [BaseJobManager]: Scheduled ESIndexOptimizer Job -foxtrot_server | INFO [2020-07-22 09:39:48,455] [main] [BaseJobManager]: Starting ConsoleHistory Manager -foxtrot_server | INFO [2020-07-22 09:39:48,455] [main] [BaseJobManager]: Scheduling ConsoleHistory Job -foxtrot_server | INFO [2020-07-22 09:39:48,455] [main] [BaseJobManager]: Scheduled ConsoleHistory Job -foxtrot_server | INFO [2020-07-22 09:39:48,456] [main] [BaseJobManager]: Starting ExpiredSessionCleaner Manager -foxtrot_server | INFO [2020-07-22 09:39:48,456] [main] [BaseJobManager]: Scheduling ExpiredSessionCleaner Job -foxtrot_server | INFO [2020-07-22 09:39:48,456] [main] [BaseJobManager]: Scheduled ExpiredSessionCleaner Job -foxtrot_server | INFO [2020-07-22 09:39:48,704] [main] [JerseyProviderInstaller]: providers = -foxtrot_server | -foxtrot_server | Exception mappers -foxtrot_server | JWTAuthenticationFailure (com.flipkart.foxtrot.server.providers.exception.AuthenticationExceptionHandler) -foxtrot_server | FoxtrotException (com.flipkart.foxtrot.server.providers.exception.FoxtrotExceptionMapper) -foxtrot_server | -foxtrot_server | Dynamic features -foxtrot_server | (com.flipkart.foxtrot.server.auth.filter.JwtAuthDynamicFeature) -foxtrot_server | -foxtrot_server | Message body writers -foxtrot_server | FlatRepresentation (com.flipkart.foxtrot.server.providers.FlatResponseCsvProvider) -foxtrot_server | Map (com.flipkart.foxtrot.server.providers.FlatResponseErrorTextProvider) -foxtrot_server | FlatRepresentation (com.flipkart.foxtrot.server.providers.FlatResponseTextProvider) -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:49,090] [main] [DropwizardResourceConfig]: The following paths were found for the configured resources: -foxtrot_server | -foxtrot_server | GET /foxtrot/google/callback (com.flipkart.foxtrot.server.resources.GoogleAuth) -foxtrot_server | GET /foxtrot/google/login (com.flipkart.foxtrot.server.resources.GoogleAuth) -foxtrot_server | GET /foxtrot/swagger (io.federecio.dropwizard.swagger.SwaggerResource) -foxtrot_server | GET /foxtrot/swagger.{type:json|yaml} (io.swagger.jaxrs.listing.ApiListingResource) -foxtrot_server | POST /foxtrot/v1/analytics (com.flipkart.foxtrot.server.resources.AnalyticsResource) -foxtrot_server | POST /foxtrot/v1/analytics/async (com.flipkart.foxtrot.server.resources.AnalyticsResource) -foxtrot_server | POST /foxtrot/v1/analytics/download (com.flipkart.foxtrot.server.resources.AnalyticsResource) -foxtrot_server | POST /foxtrot/v1/analytics/validate (com.flipkart.foxtrot.server.resources.AnalyticsResource) -foxtrot_server | POST /foxtrot/v1/async (com.flipkart.foxtrot.server.resources.AsyncResource) -foxtrot_server | GET /foxtrot/v1/async/{action}/{id} (com.flipkart.foxtrot.server.resources.AsyncResource) -foxtrot_server | GET /foxtrot/v1/auth/tokens/{tokenId} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | DELETE /foxtrot/v1/auth/tokens/{userId} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | POST /foxtrot/v1/auth/tokens/{userId} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | POST /foxtrot/v1/auth/users (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | DELETE /foxtrot/v1/auth/users/{userId} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | GET /foxtrot/v1/auth/users/{userId} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | PUT /foxtrot/v1/auth/users/{userId}/roles/grant/{role} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | PUT /foxtrot/v1/auth/users/{userId}/roles/revoke/{role} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | PUT /foxtrot/v1/auth/users/{userId}/tables/access/grant/{table} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | PUT /foxtrot/v1/auth/users/{userId}/tables/access/revoke/{table} (com.flipkart.foxtrot.server.resources.Auth) -foxtrot_server | POST /foxtrot/v1/cache/update/cardinality (com.flipkart.foxtrot.server.resources.CacheUpdateResource) -foxtrot_server | GET /foxtrot/v1/cluster/hazelcast/members (com.flipkart.foxtrot.server.resources.ClusterInfoResource) -foxtrot_server | GET /foxtrot/v1/cluster/members (com.flipkart.foxtrot.server.resources.ClusterInfoResource) -foxtrot_server | GET /foxtrot/v1/clusterhealth (com.flipkart.foxtrot.server.resources.ClusterHealthResource) -foxtrot_server | GET /foxtrot/v1/clusterhealth/indicesstats (com.flipkart.foxtrot.server.resources.ClusterHealthResource) -foxtrot_server | GET /foxtrot/v1/clusterhealth/nodestats (com.flipkart.foxtrot.server.resources.ClusterHealthResource) -foxtrot_server | GET /foxtrot/v1/consoles (com.flipkart.foxtrot.server.resources.ConsoleResource) -foxtrot_server | POST /foxtrot/v1/consoles (com.flipkart.foxtrot.server.resources.ConsoleResource) -foxtrot_server | GET /foxtrot/v1/consoles/{id} (com.flipkart.foxtrot.server.resources.ConsoleResource) -foxtrot_server | DELETE /foxtrot/v1/consoles/{id}/delete (com.flipkart.foxtrot.server.resources.ConsoleResource) -foxtrot_server | GET /foxtrot/v1/document/{table} (com.flipkart.foxtrot.server.resources.DocumentResource) -foxtrot_server | POST /foxtrot/v1/document/{table} (com.flipkart.foxtrot.server.resources.DocumentResource) -foxtrot_server | POST /foxtrot/v1/document/{table}/bulk (com.flipkart.foxtrot.server.resources.DocumentResource) -foxtrot_server | GET /foxtrot/v1/document/{table}/{id} (com.flipkart.foxtrot.server.resources.DocumentResource) -foxtrot_server | POST /foxtrot/v1/fql (com.flipkart.foxtrot.server.resources.FqlResource) -foxtrot_server | GET /foxtrot/v1/fql/download (com.flipkart.foxtrot.server.resources.FqlResource) -foxtrot_server | POST /foxtrot/v1/fql/get (com.flipkart.foxtrot.server.resources.FqlResource) -foxtrot_server | POST /foxtrot/v1/fql/save (com.flipkart.foxtrot.server.resources.FqlResource) -foxtrot_server | GET /foxtrot/v1/hbase/regions/{table}/{threshSizeInGB}/list (com.flipkart.foxtrot.server.resources.HbaseRegionsMergeResource) -foxtrot_server | GET /foxtrot/v1/hbase/regions/{table}/{threshSizeInGB}/merge/{number} (com.flipkart.foxtrot.server.resources.HbaseRegionsMergeResource) -foxtrot_server | GET /foxtrot/v1/tables (com.flipkart.foxtrot.server.resources.TableManagerResource) -foxtrot_server | POST /foxtrot/v1/tables (com.flipkart.foxtrot.server.resources.TableManagerResource) -foxtrot_server | GET /foxtrot/v1/tables/fields (com.flipkart.foxtrot.server.resources.TableFieldMappingResource) -foxtrot_server | GET /foxtrot/v1/tables/{name} (com.flipkart.foxtrot.server.resources.TableManagerResource) -foxtrot_server | PUT /foxtrot/v1/tables/{name} (com.flipkart.foxtrot.server.resources.TableManagerResource) -foxtrot_server | DELETE /foxtrot/v1/tables/{name}/delete (com.flipkart.foxtrot.server.resources.TableManagerResource) -foxtrot_server | GET /foxtrot/v1/tables/{name}/fields (com.flipkart.foxtrot.server.resources.TableFieldMappingResource) -foxtrot_server | POST /foxtrot/v1/tables/{name}/fields/update (com.flipkart.foxtrot.server.resources.TableFieldMappingResource) -foxtrot_server | GET /foxtrot/v1/util/config (com.flipkart.foxtrot.server.resources.UtilResource) -foxtrot_server | GET /foxtrot/v2/consoles (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) -foxtrot_server | POST /foxtrot/v2/consoles (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) -foxtrot_server | GET /foxtrot/v2/consoles/{id} (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) -foxtrot_server | DELETE /foxtrot/v2/consoles/{id}/delete (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) -foxtrot_server | DELETE /foxtrot/v2/consoles/{id}/old/delete (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) -foxtrot_server | GET /foxtrot/v2/consoles/{id}/old/get (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) -foxtrot_server | GET /foxtrot/v2/consoles/{id}/old/set/current (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) -foxtrot_server | GET /foxtrot/v2/consoles/{name}/old (com.flipkart.foxtrot.server.resources.ConsoleV2Resource) -foxtrot_server | -foxtrot_server | WARN [2020-07-22 09:39:49,095] [main] [Errors]: The following warnings have been detected: WARNING: A resource, Resource{"/v1/escluster", 0 child resources, 0 resource methods, 0 sub-resource locator, 0 method handler classes, 0 method handler instances}, with path "/v1/escluster" is empty. It has no resource (or sub resource) methods neither sub resource locators defined. -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:49,096] [main] [ContextHandler]: Started i.d.j.MutableServletContextHandler@6db18ce7{Application context,/,null,AVAILABLE} -foxtrot_server | INFO [2020-07-22 09:39:49,099] [main] [AdminEnvironment]: tasks = -foxtrot_server | -foxtrot_server | POST /tasks/OorTask (io.dropwizard.oor.tasks.OorTask) -foxtrot_server | POST /tasks/log-level (io.dropwizard.servlets.tasks.LogConfigurationTask) -foxtrot_server | POST /tasks/gc (io.dropwizard.servlets.tasks.GarbageCollectionTask) -foxtrot_server | POST /tasks/BirTask (io.dropwizard.oor.tasks.BirTask) -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:49,103] [main] [ContextHandler]: Started i.d.j.MutableServletContextHandler@42c32195{Admin context,/,null,AVAILABLE} -foxtrot_server | INFO [2020-07-22 09:39:49,110] [main] [AbstractConnector]: Started application@7b2c82c{HTTP/1.1,[http/1.1]}{0.0.0.0:17000} -foxtrot_server | INFO [2020-07-22 09:39:49,118] [main] [AbstractConnector]: Started admin@30fa6986{HTTP/1.1,[http/1.1]}{0.0.0.0:17001} -foxtrot_server | INFO [2020-07-22 09:39:49,118] [main] [Server]: Started @11390ms -foxtrot_server | INFO [2020-07-22 09:39:49,127] [main] [DiagnosticReporter]: Startup stats = -foxtrot_server | -foxtrot_server | GUICEY started in 1.584 s -foxtrot_server | │ -foxtrot_server | ├── [20%] CLASSPATH scanned in 310.8 ms -foxtrot_server | │ ├── scanned 309 classes -foxtrot_server | │ └── recognized 38 classes (12% of scanned) -foxtrot_server | │ -foxtrot_server | ├── [0.32%] BUNDLES processed in 5.771 ms -foxtrot_server | │ └── 3 processed -foxtrot_server | │ -foxtrot_server | ├── [77%] INJECTOR created in 1.223 s -foxtrot_server | │ ├── installers prepared in 4.723 ms -foxtrot_server | │ │ -foxtrot_server | │ ├── extensions recognized in 9.870 ms -foxtrot_server | │ │ ├── using 12 installers -foxtrot_server | │ │ └── from 309 classes -foxtrot_server | │ │ -foxtrot_server | │ └── 38 extensions installed in 5.604 ms -foxtrot_server | │ -foxtrot_server | ├── [0.25%] HK2 bridged in 4.429 ms -foxtrot_server | │ ├── using 2 jersey installers -foxtrot_server | │ └── 22 jersey extensions installed in 3.666 ms -foxtrot_server | │ -foxtrot_server | └── [2.7%] remaining 42 ms -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:49,130] [main] [DiagnosticReporter]: Options = -foxtrot_server | -foxtrot_server | Guicey (r.v.dropwizard.guice.GuiceyOptions) -foxtrot_server | ScanPackages = [com.flipkart.foxtrot] *CUSTOM -foxtrot_server | SearchCommands = false -foxtrot_server | UseCoreInstallers = true -foxtrot_server | ConfigureFromDropwizardBundles = false -foxtrot_server | BindConfigurationInterfaces = false -foxtrot_server | BindConfigurationByPath = true -foxtrot_server | InjectorStage = PRODUCTION *CUSTOM -foxtrot_server | GuiceFilterRegistration = [REQUEST] -foxtrot_server | UseHkBridge = false -foxtrot_server | -foxtrot_server | -foxtrot_server | Installers (r.v.d.g.m.i.InstallersOptions) -foxtrot_server | JerseyExtensionsManagedByGuice = true -foxtrot_server | ForceSingletonForJerseyExtensions = true -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:49,135] [main] [DiagnosticReporter]: Configuration diagnostic info = -foxtrot_server | -foxtrot_server | BUNDLES = -foxtrot_server | WebInstallersBundle (r.v.d.g.m.installer) -foxtrot_server | DiagnosticBundle (r.v.d.g.m.c.debug) -foxtrot_server | CoreInstallersBundle (r.v.d.g.m.installer) -foxtrot_server | -foxtrot_server | -foxtrot_server | INSTALLERS and EXTENSIONS in processing order = -foxtrot_server | managed (r.v.d.g.m.i.feature.ManagedInstaller) -foxtrot_server | HbaseTableConnection (c.f.f.c.d.impl.hbase) *SCAN -foxtrot_server | ElasticsearchConnection (c.f.f.c.q.impl) *SCAN -foxtrot_server | HazelcastConnection (c.f.f.c.q.impl) *SCAN -foxtrot_server | DistributedTableMetadataManager (c.f.f.c.table.impl) *SCAN -foxtrot_server | AnalyticsLoader (c.f.f.c.q.a.spi) *SCAN -foxtrot_server | DataDeletionManager (c.f.f.core.common) *SCAN -foxtrot_server | ClusterManager (c.f.f.server.cluster) *SCAN -foxtrot_server | CardinalityCalculationManager (c.f.f.c.cardinality) *SCAN -foxtrot_server | EventBusInitializer (c.f.f.c.internalevents) *SCAN -foxtrot_server | EsIndexOptimizationManager (c.f.f.c.j.optimization) *SCAN -foxtrot_server | ConsoleHistoryManager (c.f.f.s.j.consolehistory) *SCAN -foxtrot_server | DistributedSessionDataStore (c.f.f.s.a.sessionstore) *SCAN -foxtrot_server | ExpiredSessionsCleaner (c.f.f.s.j.sessioncleanup) *SCAN -foxtrot_server | jerseyprovider (r.v.d.g.m.i.f.j.p.JerseyProviderInstaller) -foxtrot_server | JwtAuthDynamicFeature (c.f.f.s.auth.filter) *SCAN -foxtrot_server | FlatResponseCsvProvider (c.f.f.s.providers) *SCAN -foxtrot_server | FlatResponseErrorTextProvider (c.f.f.s.providers) *SCAN -foxtrot_server | AuthenticationExceptionHandler (c.f.f.s.p.exception) *SCAN -foxtrot_server | FoxtrotExceptionMapper (c.f.f.s.p.exception) *SCAN -foxtrot_server | FlatResponseTextProvider (c.f.f.s.providers) *SCAN -foxtrot_server | resource (r.v.d.g.m.i.f.j.ResourceInstaller) -foxtrot_server | UtilResource (c.f.f.s.resources) *SCAN -foxtrot_server | Auth (c.f.f.s.resources) *SCAN -foxtrot_server | DocumentResource (c.f.f.s.resources) *SCAN -foxtrot_server | CacheUpdateResource (c.f.f.s.resources) *SCAN -foxtrot_server | ClusterInfoResource (c.f.f.s.resources) *SCAN -foxtrot_server | AsyncResource (c.f.f.s.resources) *SCAN -foxtrot_server | HbaseRegionsMergeResource (c.f.f.s.resources) *SCAN -foxtrot_server | TableFieldMappingResource (c.f.f.s.resources) *SCAN -foxtrot_server | ESClusterResource (c.f.f.s.resources) *SCAN -foxtrot_server | ConsoleV2Resource (c.f.f.s.resources) *SCAN -foxtrot_server | GoogleAuth (c.f.f.s.resources) *SCAN -foxtrot_server | FqlResource (c.f.f.s.resources) *SCAN -foxtrot_server | TableManagerResource (c.f.f.s.resources) *SCAN -foxtrot_server | ConsoleResource (c.f.f.s.resources) *SCAN -foxtrot_server | ClusterHealthResource (c.f.f.s.resources) *SCAN -foxtrot_server | AnalyticsResource (c.f.f.s.resources) *SCAN -foxtrot_server | healthcheck (r.v.d.g.m.i.f.h.HealthCheckInstaller) -foxtrot_server | HazelcastHealthCheck (c.f.f.s.healthcheck) *SCAN -foxtrot_server | HBaseHealthCheck (c.f.f.s.healthcheck) *SCAN -foxtrot_server | webfilter (r.v.d.g.m.i.f.web.WebFilterInstaller) -foxtrot_server | UserAuthenticationFilter (c.f.f.s.auth.filter) *SCAN -foxtrot_server | -foxtrot_server | -foxtrot_server | GUICE MODULES = -foxtrot_server | FoxtrotModule (c.f.f.server.di) -foxtrot_server | DiagnosticModule (r.v.d.g.m.c.d.DiagnosticBundle) -foxtrot_server | GuiceBootstrapModule (r.v.d.guice.module) -foxtrot_server | -foxtrot_server | INFO [2020-07-22 09:39:49,156] [main] [DiagnosticReporter]: Configuration context tree = -foxtrot_server | -foxtrot_server | APPLICATION -foxtrot_server | ├── module FoxtrotModule (c.f.f.server.di) -foxtrot_server | ├── module GuiceBootstrapModule (r.v.d.guice.module) -foxtrot_server | │ -foxtrot_server | ├── WebInstallersBundle (r.v.d.g.m.installer) -foxtrot_server | │ └── installer WebFilterInstaller (r.v.d.g.m.i.f.web) -foxtrot_server | │ -foxtrot_server | ├── DiagnosticBundle (r.v.d.g.m.c.debug) -foxtrot_server | │ └── module DiagnosticModule (r.v.d.g.m.c.d.DiagnosticBundle) -foxtrot_server | │ -foxtrot_server | ├── CoreInstallersBundle (r.v.d.g.m.installer) -foxtrot_server | │ ├── installer ManagedInstaller (r.v.d.g.m.i.feature) -foxtrot_server | │ ├── installer JerseyProviderInstaller (r.v.d.g.m.i.f.j.provider) -foxtrot_server | │ ├── installer ResourceInstaller (r.v.d.g.m.i.f.jersey) -foxtrot_server | │ └── installer HealthCheckInstaller (r.v.d.g.m.i.f.health) -foxtrot_server | │ -foxtrot_server | └── CLASSPATH SCAN -foxtrot_server | ├── extension UserAuthenticationFilter (c.f.f.s.auth.filter) -foxtrot_server | ├── extension JwtAuthDynamicFeature (c.f.f.s.auth.filter) -foxtrot_server | ├── extension DistributedSessionDataStore (c.f.f.s.a.sessionstore) -foxtrot_server | ├── extension HazelcastHealthCheck (c.f.f.s.healthcheck) -foxtrot_server | ├── extension HBaseHealthCheck (c.f.f.s.healthcheck) -foxtrot_server | ├── extension UtilResource (c.f.f.s.resources) -foxtrot_server | ├── extension Auth (c.f.f.s.resources) -foxtrot_server | ├── extension DocumentResource (c.f.f.s.resources) -foxtrot_server | ├── extension CacheUpdateResource (c.f.f.s.resources) -foxtrot_server | ├── extension ClusterInfoResource (c.f.f.s.resources) -foxtrot_server | ├── extension AsyncResource (c.f.f.s.resources) -foxtrot_server | ├── extension HbaseRegionsMergeResource (c.f.f.s.resources) -foxtrot_server | ├── extension TableFieldMappingResource (c.f.f.s.resources) -foxtrot_server | ├── extension ESClusterResource (c.f.f.s.resources) -foxtrot_server | ├── extension ConsoleV2Resource (c.f.f.s.resources) -foxtrot_server | ├── extension GoogleAuth (c.f.f.s.resources) -foxtrot_server | ├── extension FqlResource (c.f.f.s.resources) -foxtrot_server | ├── extension TableManagerResource (c.f.f.s.resources) -foxtrot_server | ├── extension ConsoleResource (c.f.f.s.resources) -foxtrot_server | ├── extension ClusterHealthResource (c.f.f.s.resources) -foxtrot_server | ├── extension AnalyticsResource (c.f.f.s.resources) -foxtrot_server | ├── extension ConsoleHistoryManager (c.f.f.s.j.consolehistory) -foxtrot_server | ├── extension ExpiredSessionsCleaner (c.f.f.s.j.sessioncleanup) -foxtrot_server | ├── extension FlatResponseCsvProvider (c.f.f.s.providers) -foxtrot_server | ├── extension FlatResponseErrorTextProvider (c.f.f.s.providers) -foxtrot_server | ├── extension AuthenticationExceptionHandler (c.f.f.s.p.exception) -foxtrot_server | ├── extension FoxtrotExceptionMapper (c.f.f.s.p.exception) -foxtrot_server | ├── extension FlatResponseTextProvider (c.f.f.s.providers) -foxtrot_server | ├── extension ClusterManager (c.f.f.server.cluster) -foxtrot_server | ├── extension HbaseTableConnection (c.f.f.c.d.impl.hbase) -foxtrot_server | ├── extension CardinalityCalculationManager (c.f.f.c.cardinality) -foxtrot_server | ├── extension AnalyticsLoader (c.f.f.c.q.a.spi) -foxtrot_server | ├── extension ElasticsearchConnection (c.f.f.c.q.impl) -foxtrot_server | ├── extension HazelcastConnection (c.f.f.c.q.impl) -foxtrot_server | ├── extension EventBusInitializer (c.f.f.c.internalevents) -foxtrot_server | ├── extension EsIndexOptimizationManager (c.f.f.c.j.optimization) -foxtrot_server | ├── extension DistributedTableMetadataManager (c.f.f.c.table.impl) -foxtrot_server | └── extension DataDeletionManager (c.f.f.core.common) -foxtrot_server | -foxtrot_server | 172.18.0.1 - - [22/Jul/2020:09:39:49 +0000] "GET /foxtrot/v1/clusterhealth HTTP/1.1" 200 3223 "http://localhost:17000/cluster/index.htm" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" 114 -foxtrot_server | 172.18.0.1 - - [22/Jul/2020:09:39:49 +0000] "GET /foxtrot/v1/clusterhealth/nodestats HTTP/1.1" 200 6879 "http://localhost:17000/cluster/index.htm" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" 114 -hbase_1 | hbase stderr | 2020-07-22 09:39:38,102 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x4ef zxid:0x21040 txntype:-1 reqpath:n/a Error Path:/hbase/namespace/default Error:KeeperErrorCode = NodeExists for /hbase/namespace/default -hbase_1 | hbase stderr | 2020-07-22 09:39:38,112 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x17375e441fa0000 type:create cxid:0x4f2 zxid:0x21042 txntype:-1 reqpath:n/a Error Path:/hbase/namespace/hbase Error:KeeperErrorCode = NodeExists for /hbase/namespace/hbase -hbase_1 | hbase stderr | 2020-07-22 09:39:38,113 INFO [hbase:41229.activeMasterManager] master.HMaster: Master has completed initialization -hbase_1 | hbase stderr | 2020-07-22 09:39:38,115 INFO [hbase:41229.activeMasterManager] quotas.MasterQuotaManager: Quota support disabled -hbase_1 | hbase stderr | 2020-07-22 09:39:38,115 INFO [hbase:41229.activeMasterManager] zookeeper.ZooKeeperWatcher: not a secure deployment, proceeding -hbase_1 | hbase stderr | 2020-07-22 09:39:42,046 INFO [HBase-Metrics2-1] impl.MetricsSystemImpl: Stopping HBase metrics system... -hbase_1 | hbase stderr | 2020-07-22 09:39:42,049 INFO [HBase-Metrics2-1] impl.MetricsSystemImpl: HBase metrics system stopped. -hbase_1 | hbase stderr | 2020-07-22 09:39:42,549 INFO [HBase-Metrics2-1] impl.MetricsConfig: loaded properties from hadoop-metrics2-hbase.properties -hbase_1 | hbase stderr | 2020-07-22 09:39:42,550 INFO [HBase-Metrics2-1] impl.MetricsSystemImpl: Scheduled snapshot period at 10 second(s). -hbase_1 | 2020-07-22 09:39:42,550 INFO [HBase-Metrics2-1] impl.MetricsSystemImpl: HBase metrics system started -hbase_1 | hbase stderr | 2020-07-22 09:39:45,165 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53410 -hbase_1 | hbase stderr | 2020-07-22 09:39:45,172 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53410 -hbase_1 | hbase stderr | 2020-07-22 09:39:45,174 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0007 with negotiated timeout 40000 for client /172.18.0.4:53410 -hbase_1 | hbase stderr | 2020-07-22 09:39:45,607 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53412 -hbase_1 | hbase stderr | 2020-07-22 09:39:45,626 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53412 -hbase_1 | hbase stderr | 2020-07-22 09:39:45,628 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0008 with negotiated timeout 40000 for client /172.18.0.4:53412 -hbase_1 | hbase stderr | 2020-07-22 09:39:48,505 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53424 -hbase_1 | hbase stderr | 2020-07-22 09:39:48,509 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53424 -hbase_1 | hbase stderr | 2020-07-22 09:39:48,510 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa0009 with negotiated timeout 40000 for client /172.18.0.4:53424 -hbase_1 | hbase stderr | 2020-07-22 09:39:48,650 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e441fa0009 -hbase_1 | hbase stderr | 2020-07-22 09:39:48,651 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxn: Closed socket connection for client /172.18.0.4:53424 which had sessionid 0x17375e441fa0009 -hbase_1 | hbase stderr | 2020-07-22 09:39:53,693 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.NIOServerCnxnFactory: Accepted socket connection from /172.18.0.4:53434 -hbase_1 | hbase stderr | 2020-07-22 09:39:53,694 INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181] server.ZooKeeperServer: Client attempting to establish new session at /172.18.0.4:53434 -hbase_1 | hbase stderr | 2020-07-22 09:39:53,696 INFO [SyncThread:0] server.ZooKeeperServer: Established session 0x17375e441fa000a with negotiated timeout 40000 for client /172.18.0.4:53434 -hbase_1 | hbase stderr | 2020-07-22 09:39:53,700 INFO [ProcessThread(sid:0 cport:-1):] server.PrepRequestProcessor: Processed session termination for sessionid: 0x17375e441fa000a -foxtrot_server | 172.18.0.1 - - [22/Jul/2020:09:39:57 +0000] "GET /foxtrot/v1/clusterhealth HTTP/1.1" 200 3223 "http://localhost:17000/cluster/index.htm" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" 13 -elasticsearch | [2020-07-22T09:39:58,244][WARN ][o.e.c.r.a.DiskThresholdMonitor] [5q9Ine5] high disk watermark [90%] exceeded on [5q9Ine5JToGzVTBOdMThhw][5q9Ine5][/usr/share/elasticsearch/data/nodes/0] free: 2.4gb[8.5%], shards will be relocated away from this node -elasticsearch | [2020-07-22T09:39:58,245][INFO ][o.e.c.r.a.DiskThresholdMonitor] [5q9Ine5] rerouting shards: [high disk watermark exceeded on one or more nodes] -foxtrot_server | 172.18.0.1 - - [22/Jul/2020:09:39:59 +0000] "GET /foxtrot/v1/clusterhealth/nodestats HTTP/1.1" 200 6881 "http://localhost:17000/cluster/index.htm" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" 35 From a2c14e2dfc59c258896c33128915de4cbbd57d1a Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 28 Jul 2021 11:12:02 +0530 Subject: [PATCH 30/37] Fixed bugs in logout flow --- .github/workflows/github-ci.yml | 23 +++++ Dockerfile | 4 +- .../core/datastore/impl/hbase/HBaseUtil.java | 40 +++------ .../foxtrot/server/InitializerCommand.java | 2 + .../auth/authprovider/AuthProvider.java | 2 + .../authprovider/impl/GoogleAuthProvider.java | 5 ++ .../authprovider/impl/IdmanAuthProvider.java | 42 ++++++--- .../auth/filter/UserAuthenticationFilter.java | 6 ++ .../AuthenticationExceptionHandler.java | 2 +- .../foxtrot/server/resources/OAuth.java | 86 ++++++++++++------- 10 files changed, 143 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/github-ci.yml diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml new file mode 100644 index 000000000..daef5b44a --- /dev/null +++ b/.github/workflows/github-ci.yml @@ -0,0 +1,23 @@ +on: + push: + branches-ignore: + - 'master' + pull_request: + branches: + - 'master' +jobs: + test_and_package: + runs-on: ubuntu-latest + continue-on-error: false + steps: + - name: Checking out code base + uses: actions/checkout@v2 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Verification + run: mvn clean verify -Dmaven.test.failure.ignore=false + continue-on-error: false diff --git a/Dockerfile b/Dockerfile index b7ed0eaa7..ccc9adafb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Santanu Sinha RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends software-properties-common RUN add-apt-repository ppa:openjdk-r/ppa && apt-get update -RUN apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates && apt-get install -y --no-install-recommends ca-certificates-java bash curl tzdata iproute2 zip unzip wget +RUN apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates ca-certificates-java bash curl tzdata iproute2 zip unzip wget EXPOSE 17000 @@ -16,5 +16,5 @@ ADD config/docker.yml docker.yml ADD foxtrot-server/target/foxtrot-server*.jar server.jar ADD scripts/local_es_setup.sh local_es_setup.sh -CMD sh -c "sleep 15 ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" +CMD sh -c "sleep 15 ; timeout 10 java -jar server.jar initialize docker.yml || true ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java index 4c72a08ea..e9eb454a1 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java @@ -17,6 +17,7 @@ package com.flipkart.foxtrot.core.datastore.impl.hbase; import com.google.common.base.Strings; +import lombok.experimental.UtilityClass; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -33,44 +34,31 @@ import java.io.File; import java.io.IOException; -public abstract class HBaseUtil { +@UtilityClass +public class HBaseUtil { private static final Logger logger = LoggerFactory.getLogger(HBaseUtil.class); - private HBaseUtil() { - } - public static void createTable(final HbaseConfig hbaseConfig, final String tableName) throws IOException { - HTableDescriptor hTableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); + final TableName name = TableName.valueOf(tableName); + HTableDescriptor hTableDescriptor = new HTableDescriptor(name); HColumnDescriptor columnDescriptor = new HColumnDescriptor("d"); columnDescriptor.setCompressionType(Compression.Algorithm.GZ); hTableDescriptor.addFamily(columnDescriptor); Configuration configuration = HBaseUtil.create(hbaseConfig); - Connection connection = ConnectionFactory.createConnection(configuration); - Admin hBaseAdmin = null; - try { - hBaseAdmin = connection.getAdmin(); - hBaseAdmin.createTable(hTableDescriptor); + + try(Connection connection = ConnectionFactory.createConnection(configuration); + Admin hBaseAdmin = connection.getAdmin()) { + if(hBaseAdmin.tableExists(name)) { + logger.info("Table {} exists", tableName); + } + else { + hBaseAdmin.createTable(hTableDescriptor); + } } catch (Exception e) { logger.error("Could not create table: " + tableName, e); } - finally { - try { - if (hBaseAdmin != null) { - hBaseAdmin.close(); - } - } - catch (Exception e) { - logger.error("Error closing hbase admin", e); - } - try { - connection.close(); - } - catch (Exception e) { - logger.error("Error closing hbase connection", e); - } - } } public static Configuration create(final HbaseConfig hbaseConfig) throws IOException { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java index 2319105c7..14326dae0 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java @@ -79,6 +79,8 @@ protected void run(Bootstrap bootstrap, Namespace na logger.info("Creating hbase table"); HBaseUtil.createTable(configuration.getHbase(), configuration.getHbase() .getTableName()); + + logger.info("Initialization complete"); } private void createMetaIndex(final ElasticsearchConnection connection, final String indexName, int replicaCount) { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java index 48cfecb3b..b4e911c32 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java @@ -22,4 +22,6 @@ default boolean isPreregistrationRequired() { Optional login(String authCode, String sessionId); Optional authenticate(AuthInfo authInfo); + + boolean logout(String sessionId); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java index 3ca298544..56c47944e 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java @@ -201,6 +201,11 @@ public Optional authenticate(AuthInfo authInfo) { return Optional.of(new AuthenticatedInfo(token, user)); } + @Override + public boolean logout(String sessionId) { + return credentialsStorage.deleteToken(sessionId); + } + @Override public boolean isPreregistrationRequired() { return false; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java index bce03fd64..8c04719f3 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java @@ -7,6 +7,7 @@ import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; import com.flipkart.foxtrot.server.auth.authprovider.AuthType; import com.flipkart.foxtrot.server.auth.authprovider.IdType; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import io.appform.idman.model.IdmanUser; import io.appform.idman.model.TokenInfo; @@ -65,7 +66,7 @@ public String redirectionURL(String sessionId) { @Override @SneakyThrows public Optional login(String authCode, String sessionId) { - return callIdmanServer("authorization_code", authCode, + return remoteTokenCall("authorization_code", authCode, ti -> { val idmanUser = ti.getUser(); val expiry = Date.from(Instant.now().plusSeconds(ti.getExpiry())); @@ -80,14 +81,9 @@ public Optional login(String authCode, String sessionId) { @Override @SneakyThrows public Optional authenticate(AuthInfo authInfo) { - return refreshToken(authInfo.accept(TokenAuthInfo::getToken)); - } - - @SneakyThrows - private Optional refreshToken(String token) { - return callIdmanServer( + return remoteTokenCall( "refresh_token", - token, + authInfo.accept(TokenAuthInfo::getToken), ti -> { val idmanUser = ti.getUser(); val expiry = Date.from(Instant.now().plusSeconds(ti.getExpiry())); @@ -107,6 +103,16 @@ private Optional refreshToken(String token) { }); } + @Override + @SneakyThrows + public boolean logout(String sessionId) { + if(Strings.isNullOrEmpty(sessionId)) { + log.warn("Empty token send for logout"); + return true; + } + return remoteTokenRevokeCall(sessionId); + } + private Set mapToFoxtrotRoles(final IdmanUser idmanUser) { switch (idmanUser.getRole()) { case "FOXTROT_ADMIN": { @@ -127,15 +133,14 @@ private boolean isSystemUser(final IdmanUser user) { return user.getRole().equals("FOXTROT_ADMIN"); } - private Optional callIdmanServer( + private Optional remoteTokenCall( String callMode, String param, Function> handler) throws URISyntaxException, IOException { val post = new HttpPost(new URIBuilder(config.getIdmanEndpoint()) .setPath("/apis/oauth2/token") .build()); - val fromBuilder = ImmutableList.builder(); - val form = fromBuilder + val form = ImmutableList.builder() .add(new BasicNameValuePair(callMode.equals("authorization_code") ? "code" : "refresh_token", param)) @@ -154,4 +159,19 @@ private Optional callIdmanServer( } } + private boolean remoteTokenRevokeCall(String token) throws URISyntaxException, IOException { + val post = new HttpPost(new URIBuilder(config.getIdmanEndpoint()) + .setPath("/apis/oauth2/revoke") + .build()); + val params = ImmutableList.builder() + .add(new BasicNameValuePair("client_id", config.getClientId())) + .add(new BasicNameValuePair("client_secret", config.getClientSecret())) + .add(new BasicNameValuePair("token", token)) + .build(); + post.setEntity(new UrlEncodedFormEntity(params)); + try (val response = client.execute(post)) { + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; + } + } + } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java index 2c0c2426f..e54addf58 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java @@ -75,11 +75,17 @@ public void doFilter( chain.doFilter(request, response); return; } + else { + log.info("No principal "); + } } catch (AuthenticationException e) { log.error("Jwt validation failure: ", e); } } + else { + log.debug("No token in request"); + } val referrer = httpRequest.getHeader(org.apache.http.HttpHeaders.REFERER); val source = Strings.isNullOrEmpty(referrer) ? requestURI : referrer; httpResponse.addCookie(new Cookie("redirection", source)); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java index 85845391b..4c33442d7 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java @@ -19,7 +19,7 @@ public class AuthenticationExceptionHandler implements ExceptionMapper Date: Wed, 28 Jul 2021 15:31:59 +0530 Subject: [PATCH 31/37] Created separate configs etc for different auth modes. Made auth config completely optional --- .gitignore | 1 + Dockerfile | 5 +- config/docker-gauth.yml | 89 +++++++++++++++++++ config/docker-idman.yml | 89 +++++++++++++++++++ config/docker.yml | 18 ---- docker-compose-gauth.yml | 59 ++++++++++++ docker-compose-idman.yml | 57 ++++++++++++ docker-compose.yml | 7 +- .../foxtrot/server/auth/AuthConfig.java | 3 +- .../foxtrot/server/auth/JwtConfig.java | 4 +- .../auth/authprovider/AuthConfigVisitor.java | 3 + .../server/auth/authprovider/AuthType.java | 1 + .../ConfiguredAuthProviderFactory.java | 10 ++- .../auth/authprovider/NoneAuthProvider.java | 37 ++++++++ .../impl/NoneAuthProviderConfig.java | 20 +++++ .../config/FoxtrotServerConfiguration.java | 7 +- .../foxtrot/server/di/FoxtrotModule.java | 35 +++++--- startup.sh | 28 ++++++ 18 files changed, 421 insertions(+), 52 deletions(-) create mode 100755 config/docker-gauth.yml create mode 100755 config/docker-idman.yml create mode 100644 docker-compose-gauth.yml create mode 100644 docker-compose-idman.yml create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/NoneAuthProvider.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/NoneAuthProviderConfig.java create mode 100755 startup.sh diff --git a/.gitignore b/.gitignore index 814b05682..29fbc3916 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ target/* config/test.yml gauth-creds-env.sh +idman-creds-env.sh diff --git a/Dockerfile b/Dockerfile index ccc9adafb..522915424 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,9 +12,10 @@ EXPOSE 5701 VOLUME /var/log/foxtrot-server -ADD config/docker.yml docker.yml +ADD config/docker* config/ ADD foxtrot-server/target/foxtrot-server*.jar server.jar ADD scripts/local_es_setup.sh local_es_setup.sh +ADD startup.sh startup.sh -CMD sh -c "sleep 15 ; timeout 10 java -jar server.jar initialize docker.yml || true ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" +CMD ./startup.sh diff --git a/config/docker-gauth.yml b/config/docker-gauth.yml new file mode 100755 index 000000000..1d506d94d --- /dev/null +++ b/config/docker-gauth.yml @@ -0,0 +1,89 @@ +server: + applicationConnectors: + - type: http + port: 17000 + adminConnectors: + - type: http + port: 17001 + +swaggerScheme: ${SWAGGER_SCHEME} + +elasticsearch: + hosts: + - ${ELASTICSEARCH_HOST} + cluster: ${ELASTICSEARCH_CLUSTER_NAME} + tableNamePrefix: ${ELASTICSEARCH_TABLE_NAME_PREFIX} + port: ${ELASTICSEARCH_PORT} + connectionType: ${ELASTICSEARCH_PROTOCOL} + +hbase: + secure : false + tableName: ${HBASE_TABLE_NAME} + hbaseZookeeperQuorum: ${HBASE_ZOOKEEPER_QUORUM} + hbaseZookeeperClientPort: ${HBASE_ZOOKEEPER_CLIENT_PORT} + +cluster: + name: foxtrot + discovery: + type: ${DISCOVERY_TYPE} + disableMulticast: true + members: ["localhost:5701"] + +cacheConfig: + maxIdleSeconds: 15 + timeToLiveSeconds: 15 + +logging: + level: INFO + loggers: + com.flipkart.foxtrot.core.querystore.actions: DEBUG + org.apache.hadoop.hbase.zookeeper: WARN + org.apache.zookeeper: WARN + org.apache.hadoop.hbase.client: WARN + appenders: + - type: console + threshold: TRACE + timeZone: IST + logFormat: "%(%-5level) [%date] %X{TRACE-ID} [%thread] [%logger{0}]: %message%n" + +cardinality: + enabled: true + batchSize: 10 + active: true + interval: 86400 + initialDelay: 6 + maxCardinality: 20000 + +deletionconfig: + active: true + interval: 86400 + initialdelay: 60 + +esIndexOptimizationConfig: + active: true + interval: 86400 + initialDelay: 6 + +consoleHistoryConfig: + active: true + interval: 86400 + initialDelay: 10 + +sessionCleanupConfig: + active: true + interval: 86400 + initialDelay: 10 + +auth: + enabled: true + jwt: + issuerId: foxtrot-server + privateKey: ${PRIVATE_KEY} + sessionDuration: 15d + + provider: + type: OAUTH_GOOGLE + clientId: ${GOOGLE_CLIENT_ID} + clientSecret: ${GOOGLE_CLIENT_SECRET} + server: ${GOOGLE_CALLBACK_HOST_PORT} + secureEndpoint: ${GOOGLE_SECURE_ENDPOINT} \ No newline at end of file diff --git a/config/docker-idman.yml b/config/docker-idman.yml new file mode 100755 index 000000000..f3ad611d8 --- /dev/null +++ b/config/docker-idman.yml @@ -0,0 +1,89 @@ +server: + applicationConnectors: + - type: http + port: 17000 + adminConnectors: + - type: http + port: 17001 + +swaggerScheme: ${SWAGGER_SCHEME} + +elasticsearch: + hosts: + - ${ELASTICSEARCH_HOST} + cluster: ${ELASTICSEARCH_CLUSTER_NAME} + tableNamePrefix: ${ELASTICSEARCH_TABLE_NAME_PREFIX} + port: ${ELASTICSEARCH_PORT} + connectionType: ${ELASTICSEARCH_PROTOCOL} + +hbase: + secure : false + tableName: ${HBASE_TABLE_NAME} + hbaseZookeeperQuorum: ${HBASE_ZOOKEEPER_QUORUM} + hbaseZookeeperClientPort: ${HBASE_ZOOKEEPER_CLIENT_PORT} + +cluster: + name: foxtrot + discovery: + type: ${DISCOVERY_TYPE} + disableMulticast: true + members: ["localhost:5701"] + +cacheConfig: + maxIdleSeconds: 15 + timeToLiveSeconds: 15 + +logging: + level: INFO + loggers: + com.flipkart.foxtrot.core.querystore.actions: DEBUG + org.apache.hadoop.hbase.zookeeper: WARN + org.apache.zookeeper: WARN + org.apache.hadoop.hbase.client: WARN + appenders: + - type: console + threshold: TRACE + timeZone: IST + logFormat: "%(%-5level) [%date] %X{TRACE-ID} [%thread] [%logger{0}]: %message%n" + +cardinality: + enabled: true + batchSize: 10 + active: true + interval: 86400 + initialDelay: 6 + maxCardinality: 20000 + +deletionconfig: + active: true + interval: 86400 + initialdelay: 60 + +esIndexOptimizationConfig: + active: true + interval: 86400 + initialDelay: 6 + +consoleHistoryConfig: + active: true + interval: 86400 + initialDelay: 10 + +sessionCleanupConfig: + active: true + interval: 86400 + initialDelay: 10 + +auth: + enabled: true + jwt: + issuerId: foxtrot-server + privateKey: ${PRIVATE_KEY} + sessionDuration: 15d + + provider: + type: OAUTH_IDMAN + idmanEndpoint: ${IDMAN_ENDPOINT} + clientId: FOXTROT + clientSecret: 8f4bcf45-2909-42e0-9bba-f46351bb0e6d + serverEndpoint: "http://localhost:17000" diff --git a/config/docker.yml b/config/docker.yml index 8f515f1c7..7cf359374 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -74,21 +74,3 @@ sessionCleanupConfig: interval: 86400 initialDelay: 10 -auth: - enabled: ${AUTH_ENABLED} - jwt: - issuerId: foxtrot-server - privateKey: ${PRIVATE_KEY} - sessionDuration: 15d - - provider: -# type: OAUTH_GOOGLE -# clientId: ${GOOGLE_CLIENT_ID} -# clientSecret: ${GOOGLE_CLIENT_SECRET} -# server: ${GOOGLE_CALLBACK_HOST_PORT} -# secureEndpoint: ${GOOGLE_SECURE_ENDPOINT} - type: OAUTH_IDMAN - idmanEndpoint: https://ssidman.loca.lt - clientId: FOXTROT - clientSecret: 8f4bcf45-2909-42e0-9bba-f46351bb0e6d - serverEndpoint: "http://localhost:17000" diff --git a/docker-compose-gauth.yml b/docker-compose-gauth.yml new file mode 100644 index 000000000..dd9d1b6d0 --- /dev/null +++ b/docker-compose-gauth.yml @@ -0,0 +1,59 @@ +version: '2' +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.8 + container_name: elasticsearch + hostname: elasticsearch + environment: + - cluster.name=elasticsearch + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + command: elasticsearch -Ehttp.cors.enabled=true + volumes: + - /elasticsearch/data + + hbase: + image: hyness/hbase-rest-standalone + hostname: hbase + volumes: + - /data/hbase + - /data/zookeeper + + foxtrot-server: + depends_on: + - elasticsearch + - hbase + container_name: foxtrot_server + build: . + ports: + - "17000:17000" + - "17001:17001" + - "5701:5701" + volumes: + - /var/log/foxtrot-server + environment: + - HOST=localhost + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9300 + - ELASTICSEARCH_CLUSTER_NAME=elasticsearch + - ELASTICSEARCH_TABLE_NAME_PREFIX=foxtrot + - ELASTICSEARCH_PORT="9200" + - ELASTICSEARCH_PROTOCOL=http + - HBASE_TABLE_NAME=foxtrot + - HBASE_ZOOKEEPER_QUORUM=hbase + - HBASE_ZOOKEEPER_CLIENT_PORT=2181 + - HAZELCAST_CLUSTER_NAME=foxtrot + - GOOGLE_CALLBACK_HOST_PORT=localhost:17000 + - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} + - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} + - GOOGLE_SECURE_ENDPOINT=false + - DISCOVERY_TYPE=foxtrot_simple + - SWAGGER_SCHEME="http" + - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" + - STARTUP_DELAY=15 + - CONFIG_PATH=/config/docker-gauth.yml + diff --git a/docker-compose-idman.yml b/docker-compose-idman.yml new file mode 100644 index 000000000..443d2e6d2 --- /dev/null +++ b/docker-compose-idman.yml @@ -0,0 +1,57 @@ +version: '2' +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.8 + container_name: elasticsearch + hostname: elasticsearch + environment: + - cluster.name=elasticsearch + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + command: elasticsearch -Ehttp.cors.enabled=true + volumes: + - /elasticsearch/data + + hbase: + image: hyness/hbase-rest-standalone + hostname: hbase + volumes: + - /data/hbase + - /data/zookeeper + + foxtrot-server: + depends_on: + - elasticsearch + - hbase + container_name: foxtrot_server + build: . + ports: + - "17000:17000" + - "17001:17001" + - "5701:5701" + volumes: + - /var/log/foxtrot-server + environment: + - HOST=localhost + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9300 + - ELASTICSEARCH_CLUSTER_NAME=elasticsearch + - ELASTICSEARCH_TABLE_NAME_PREFIX=foxtrot + - ELASTICSEARCH_PORT="9200" + - ELASTICSEARCH_PROTOCOL=http + - HBASE_TABLE_NAME=foxtrot + - HBASE_ZOOKEEPER_QUORUM=hbase + - HBASE_ZOOKEEPER_CLIENT_PORT=2181 + - HAZELCAST_CLUSTER_NAME=foxtrot + - DISCOVERY_TYPE=foxtrot_simple + - SWAGGER_SCHEME="http" + - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" + - STARTUP_DELAY=15 + - CONFIG_PATH=/config/docker-idman.yml + - IDMAN_ENDPOINT=${IDMAN_ENDPOINT} + - IDMAN_SECRET=${IDMAN_SECRET} + diff --git a/docker-compose.yml b/docker-compose.yml index 0b9544f73..65ca5ce1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,11 +57,6 @@ services: - HBASE_ZOOKEEPER_QUORUM=hbase - HBASE_ZOOKEEPER_CLIENT_PORT=2181 - HAZELCAST_CLUSTER_NAME=foxtrot - - AUTH_ENABLED=true - - GOOGLE_CALLBACK_HOST_PORT=localhost:17000 - - GOOGLE_CLIENT_ID=dummy - - GOOGLE_CLIENT_SECRET=dummy - - GOOGLE_SECURE_ENDPOINT=false - DISCOVERY_TYPE=foxtrot_simple - SWAGGER_SCHEME="http" - - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" + - STARTUP_DELAY=15 diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java index 84c28f4fc..6d238c2b5 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java @@ -1,6 +1,7 @@ package com.flipkart.foxtrot.server.auth; import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.NoneAuthProviderConfig; import lombok.Data; import javax.validation.Valid; @@ -19,5 +20,5 @@ public class AuthConfig { @NotNull @Valid - public AuthProviderConfig provider; + public AuthProviderConfig provider = new NoneAuthProviderConfig(); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java index db2835598..bd870082b 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java @@ -16,11 +16,11 @@ public class JwtConfig { @NotNull @NotEmpty - private String privateKey; + private String privateKey = "useless_default_key"; @NotNull @NotEmpty - private String issuerId; + private String issuerId = "foxtrot"; @NotEmpty @NotNull diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java index 41dac7f9a..38684bedf 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java @@ -2,6 +2,7 @@ import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.NoneAuthProviderConfig; /** * @@ -10,4 +11,6 @@ public interface AuthConfigVisitor { T visit(GoogleAuthProviderConfig googleAuthConfig); T visit(IdmanAuthProviderConfig idmanAuthProviderConfig); + + T visit(NoneAuthProviderConfig noneAuthProviderConfig); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java index 4e3d660d2..913dc16bd 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java @@ -4,6 +4,7 @@ * */ public enum AuthType { + NONE, OAUTH_GOOGLE, OAUTH_IDMAN } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java index 7ede8768b..a0505b1d7 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java @@ -3,10 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.server.auth.AuthStore; -import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProvider; -import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; -import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProvider; -import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.*; /** * @@ -31,6 +28,11 @@ public AuthProvider visit(GoogleAuthProviderConfig googleAuthConfig) { public AuthProvider visit(IdmanAuthProviderConfig idmanAuthProviderConfig) { return new IdmanAuthProvider(idmanAuthProviderConfig, mapper); } + + @Override + public AuthProvider visit(NoneAuthProviderConfig noneAuthProviderConfig) { + return new NoneAuthProvider(); + } }); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/NoneAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/NoneAuthProvider.java new file mode 100644 index 000000000..5aad4ced9 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/NoneAuthProvider.java @@ -0,0 +1,37 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.flipkart.foxtrot.server.auth.AuthInfo; +import com.flipkart.foxtrot.server.auth.AuthenticatedInfo; +import com.flipkart.foxtrot.server.auth.Token; + +import java.util.Optional; + +/** + * + */ +public class NoneAuthProvider implements AuthProvider { + @Override + public AuthType type() { + return AuthType.NONE; + } + + @Override + public String redirectionURL(String sessionId) { + throw new IllegalStateException("Redirection called on NONE auth mode"); + } + + @Override + public Optional login(String authCode, String sessionId) { + return Optional.empty(); + } + + @Override + public Optional authenticate(AuthInfo authInfo) { + return Optional.empty(); + } + + @Override + public boolean logout(String sessionId) { + return true; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/NoneAuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/NoneAuthProviderConfig.java new file mode 100644 index 000000000..8715c36c1 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/NoneAuthProviderConfig.java @@ -0,0 +1,20 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.flipkart.foxtrot.server.auth.authprovider.AuthConfigVisitor; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; + +/** + * + */ +public class NoneAuthProviderConfig extends AuthProviderConfig { + + public NoneAuthProviderConfig() { + super(AuthType.NONE); + } + + @Override + public T accept(AuthConfigVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java index 4a0b379a3..1d95b16fb 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java @@ -26,14 +26,13 @@ import com.flipkart.foxtrot.core.querystore.impl.CacheConfig; import com.flipkart.foxtrot.core.querystore.impl.ClusterConfig; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConfig; -import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.core.reroute.ClusterRerouteConfig; +import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.server.jobs.consolehistory.ConsoleHistoryConfig; import com.flipkart.foxtrot.server.jobs.sessioncleanup.SessionCleanupConfig; import com.foxtrot.flipkart.translator.config.SegregationConfiguration; import com.foxtrot.flipkart.translator.config.TranslatorConfig; import io.dropwizard.Configuration; -import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -94,18 +93,16 @@ public class FoxtrotServerConfiguration extends Configuration { private String swaggerScheme; - @Builder.Default private TranslatorConfig translatorConfig = new TranslatorConfig(); @Valid - @Builder.Default private TextNodeRemoverConfiguration textNodeRemover = new TextNodeRemoverConfiguration(); private ClusterRerouteConfig clusterRerouteConfig; @Valid @NotNull - private AuthConfig auth; + private AuthConfig auth = new AuthConfig(); public FoxtrotServerConfiguration() { this.hbase = new HbaseConfig(); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java index 68ec99d8f..cd36f6c7f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java @@ -109,7 +109,8 @@ protected void configure() { .to(StrSubstitutorEmailBodyBuilder.class); bind(TableManager.class) .to(FoxtrotTableManager.class); - bind(new TypeLiteral>() {}).toProvider(HealthcheckListProvider.class); + bind(new TypeLiteral>() { + }).toProvider(HealthcheckListProvider.class); bind(AuthStore.class) .to(ESAuthStore.class); bind(SessionDataStore.class) @@ -130,7 +131,7 @@ public ElasticsearchConfig esConfig(FoxtrotServerConfiguration configuration) { @Provides @Singleton - public TranslatorConfig getTranslatorConfig(FoxtrotServerConfiguration configuration){ + public TranslatorConfig getTranslatorConfig(FoxtrotServerConfiguration configuration) { return configuration.getTranslatorConfig(); } @@ -144,16 +145,16 @@ public ClusterConfig clusterConfig(FoxtrotServerConfiguration configuration) { @Singleton public CardinalityConfig cardinalityConfig(FoxtrotServerConfiguration configuration) { return null == configuration.getCardinality() - ? new CardinalityConfig("false", String.valueOf(ElasticsearchUtils.DEFAULT_SUB_LIST_SIZE)) - : configuration.getCardinality(); + ? new CardinalityConfig("false", String.valueOf(ElasticsearchUtils.DEFAULT_SUB_LIST_SIZE)) + : configuration.getCardinality(); } @Provides @Singleton public EsIndexOptimizationConfig esIndexOptimizationConfig(FoxtrotServerConfiguration configuration) { return null == configuration.getEsIndexOptimizationConfig() - ? new EsIndexOptimizationConfig() - : configuration.getEsIndexOptimizationConfig(); + ? new EsIndexOptimizationConfig() + : configuration.getEsIndexOptimizationConfig(); } @Provides @@ -166,16 +167,16 @@ public DataDeletionManagerConfig dataDeletionManagerConfig(FoxtrotServerConfigur @Singleton public ConsoleHistoryConfig consoleHistoryConfig(FoxtrotServerConfiguration configuration) { return null == configuration.getConsoleHistoryConfig() - ? new ConsoleHistoryConfig() - : configuration.getConsoleHistoryConfig(); + ? new ConsoleHistoryConfig() + : configuration.getConsoleHistoryConfig(); } @Provides @Singleton public SessionCleanupConfig sessionCleanupConfig(FoxtrotServerConfiguration configuration) { return null == configuration.getSessionCleanupConfig() - ? new SessionCleanupConfig() - : configuration.getSessionCleanupConfig(); + ? new SessionCleanupConfig() + : configuration.getSessionCleanupConfig(); } @Provides @@ -212,7 +213,9 @@ public List provideMutators( @Provides @Singleton - public List actionExecutionObservers(CacheManager cacheManager, InternalEventBus eventBus) { + public List actionExecutionObservers( + CacheManager cacheManager, + InternalEventBus eventBus) { return ImmutableList.builder() .add(new MetricRecorder()) .add(new ResponseCacheUpdater(cacheManager)) @@ -269,12 +272,15 @@ public GoogleAuthProviderConfig googleAuthProviderConfig(FoxtrotServerConfigurat @Singleton public AuthProvider authProvider( FoxtrotServerConfiguration configuration, + AuthConfig authConfig, Environment environment, Injector injector) { - val authType = configuration.getAuth().getProvider().getType(); + val authType = authConfig.getProvider().getType(); AuthStore authStore = null; switch (authType) { - + case NONE: { + break; + } case OAUTH_GOOGLE: authStore = injector.getInstance(ESAuthStore.class); break; @@ -335,7 +341,8 @@ public Authorizer authorizer( @Singleton public ElasticsearchTuningConfig provideElasticsearchTuningConfig(FoxtrotServerConfiguration configuration) { return Objects.nonNull(configuration.getElasticsearchTuningConfig()) - ? configuration.getElasticsearchTuningConfig() : new ElasticsearchTuningConfig(); + ? configuration.getElasticsearchTuningConfig() + : new ElasticsearchTuningConfig(); } } diff --git a/startup.sh b/startup.sh new file mode 100755 index 000000000..7cc3c09af --- /dev/null +++ b/startup.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +FOXTROT_CONFIG_PATH="/config/docker.yml" + +if [ -z "${CONFIG_PATH}" ]; then + : +else + FOXTROT_CONFIG_PATH=${CONFIG_PATH} +fi +echo "Using config path: ${FOXTROT_CONFIG_PATH}" + + +if [ -z "${STARTUP_DELAY}" ]; then + echo "No delay specified. To wait for stores to initialize, please provide a delay in the STARTUP_DELAY variable" +else + echo "Sleeping for ${STARTUP_DELAY} seconds" + sleep ${STARTUP_DELAY} +fi + +if [ -z "${SKIP_INITIALIZATION}" ]; then + timeout 10 timeout 10 java -jar server.jar initialize "${FOXTROT_CONFIG_PATH}" +else + echo "Skipping initialization for foxtrot data stores" +fi + +CMD="java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server ${FOXTROT_CONFIG_PATH}" +echo "Executing server command: ${CMD}" +${CMD} \ No newline at end of file From 7d0164b712838e3ff515d595f0050cb1f2a458c2 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 28 Jul 2021 16:06:20 +0530 Subject: [PATCH 32/37] Trying to gix github action --- .github/workflows/github-ci.yml | 4 ++++ pom.xml | 7 ------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index daef5b44a..d4f25e5f8 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -18,6 +18,10 @@ jobs: with: java-version: 1.8 + - name: Pull ryuk + run: docker pull quay.io/testcontainers/ryuk:0.2.2 + continue-on-error: false + - name: Verification run: mvn clean verify -Dmaven.test.failure.ignore=false continue-on-error: false diff --git a/pom.xml b/pom.xml index 9d01fdce2..64f49a7c0 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,6 @@ http://localhost:9200 1.16.16 4.2.1 - 6.3.1-7 1.0.4 24.1.1-jre src/main @@ -162,12 +161,6 @@ - - local - - 6.3.1-7 - - From 39fc6b9642db07b69506d40470a3e84a0cd3df71 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 28 Jul 2021 18:03:32 +0530 Subject: [PATCH 33/37] Added central snapshots --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 64f49a7c0..47dcdb7c4 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,12 @@ bintray https://jcenter.bintray.com + + snapshots-repo + https://oss.sonatype.org/content/repositories/snapshots + false + true + From a30d75d18a763b0854c3ae608d9ab89617260b14 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Tue, 3 Aug 2021 10:38:47 +0530 Subject: [PATCH 34/37] Addressed review comments --- foxtrot-server/pom.xml | 2 +- .../flipkart/foxtrot/server/auth/AuthStore.java | 4 ++-- .../flipkart/foxtrot/server/auth/ESAuthStore.java | 12 ++++++------ .../foxtrot/server/auth/IdmanAuthStore.java | 4 ++-- .../auth/authprovider/impl/GoogleAuthProvider.java | 14 +++++++++----- .../auth/authprovider/impl/IdmanAuthProvider.java | 2 +- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index dd05ab1b8..df24542e3 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -99,7 +99,7 @@ io.appform.idman idman-model - 1.0-SNAPSHOT + 1.0 * diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java index c55b820f5..96646c4ca 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java @@ -20,9 +20,9 @@ public interface AuthStore { Optional getUser(final String userId); - boolean deleteUser(final String id); + boolean deleteUser(final String userId); - boolean updateUser(final String id, UnaryOperator mutator); + boolean updateUser(final String userId, UnaryOperator mutator); default boolean grantRole(final String userId, final FoxtrotRole role) { return updateUser(userId, user -> { diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java index 57d2242c7..2c965df08 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -72,14 +72,14 @@ public Optional getUser(String userId) { @SneakyThrows @Override - public boolean deleteUser(String id) { + public boolean deleteUser(String userId) { boolean status = connection.getClient() - .delete(new DeleteRequest(USERS_INDEX, USER_TYPE, id) + .delete(new DeleteRequest(USERS_INDEX, USER_TYPE, userId) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) .status() == RestStatus.OK; if (status) { - val count = deleteTokensForUser(id); - log.debug("User {} deleted and {} existing tokens deleted with that.", id, count); + val count = deleteTokensForUser(userId); + log.debug("User {} deleted and {} existing tokens deleted with that.", userId, count); } return status; } @@ -87,8 +87,8 @@ public boolean deleteUser(String id) { @Override @SneakyThrows public boolean updateUser( - String id, UnaryOperator mutator) { - val user = getUser(id).orElse(null); + String userId, UnaryOperator mutator) { + val user = getUser(userId).orElse(null); if (null == user) { return false; } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java index 9a9e27ad2..5af4c9e4a 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java @@ -23,13 +23,13 @@ public Optional getUser(String userId) { } @Override - public boolean deleteUser(String id) { + public boolean deleteUser(String userId) { return false; } @Override public boolean updateUser( - String id, UnaryOperator mutator) { + String userId, UnaryOperator mutator) { return false; } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java index 56c47944e..ae2c91184 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java @@ -70,15 +70,13 @@ public GoogleAuthProvider( case HTTP: { Preconditions.checkArgument(!Strings.isNullOrEmpty(googleAuthConfig.getProxyHost())); proxy = new Proxy(Proxy.Type.HTTP, - new InetSocketAddress(googleAuthConfig.getProxyHost(), - googleAuthConfig.getProxyPort())); + proxyAddress(googleAuthConfig)); break; } case SOCKS: Preconditions.checkArgument(!Strings.isNullOrEmpty(googleAuthConfig.getProxyHost())); - proxy = new Proxy(Proxy.Type.HTTP, - new InetSocketAddress(googleAuthConfig.getProxyHost(), - googleAuthConfig.getProxyPort())); + proxy = new Proxy(Proxy.Type.SOCKS, + proxyAddress(googleAuthConfig)); break; } } @@ -228,4 +226,10 @@ private JwtConsumer buildJwtConsumer() { .build(); } + + private InetSocketAddress proxyAddress(GoogleAuthProviderConfig googleAuthConfig) { + return new InetSocketAddress(googleAuthConfig.getProxyHost(), + googleAuthConfig.getProxyPort()); + } + } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java index 8c04719f3..1e231c8ff 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java @@ -119,7 +119,7 @@ private Set mapToFoxtrotRoles(final IdmanUser idmanUser) { return EnumSet.allOf(FoxtrotRole.class); } case "FOXTROT_HUMAN_USER": { - return EnumSet.of(FoxtrotRole.CONSOLE, FoxtrotRole.QUERY, FoxtrotRole.INGEST); + return EnumSet.of(FoxtrotRole.CONSOLE, FoxtrotRole.QUERY); } case "FOXTROT_SYSTEM_USER": { return EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.INGEST); From 1291a487d1e8716e33b5cc3fa4f6be0108b20d34 Mon Sep 17 00:00:00 2001 From: Nitish Goyal Date: Tue, 3 Aug 2021 13:14:29 +0530 Subject: [PATCH 35/37] Build fix --- .../flipkart/foxtrot/common/query/MultiQueryRequest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiQueryRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiQueryRequest.java index f2e58ea27..cdce2e9d2 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiQueryRequest.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiQueryRequest.java @@ -15,14 +15,11 @@ * limitations under the License. */ -import com.collections.CollectionUtils; import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionRequestVisitor; import com.flipkart.foxtrot.common.Opcodes; -import lombok.Data; -import org.junit.Assert; - import java.util.Map; +import lombok.Data; /*** Created by nitish.goyal on 22/08/18 @@ -38,7 +35,6 @@ public MultiQueryRequest() { public MultiQueryRequest(Map requests) { super(Opcodes.MULTI_QUERY); - Assert.assertTrue(CollectionUtils.isNotEmpty(requests)); this.requests = requests; } From 229ccb89556a1a317882ec0abdd824a82429db70 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Tue, 25 Jan 2022 09:52:13 +0530 Subject: [PATCH 36/37] Added more infor to README --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b5f2fe2a..7c5610cf0 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,15 @@ Check the [Wiki](https://github.com/Flipkart/foxtrot/wiki/Introduction) for deta Version ---- +6.3.1-9 -0.1 +Docker +------ +Docker can be found on [DockerHub](https://hub.docker.com/layers/santanusinha/foxtrot/6.3.1-9/images/sha256-01cb327eb0353d31874681ee9ece4df8993b2152dfd3fa7279a31e3d7e32ee7e?context=explore) + +Docker can be customised using environment variables. Refer [here](https://github.com/Flipkart/foxtrot/blob/simple_auth/config/docker.yml) for the variables that need to be passed. + +Volume mount the config file in a docker. And pass in the full path to file `CONFIG_PATH` environment variable. Tech ----------- From f45f4d588822d70819f6436ab63abba00a4b6db9 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Tue, 25 Jan 2022 15:29:10 +0530 Subject: [PATCH 37/37] Hazelcast version bump --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f92f18c44..803b621b9 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ ${project.basedir}/target/jacoco.exec java **com/flipkart/foxtrot/common/**/* - 4.0.2 + 4.0.5