Skip to content

Commit

Permalink
Issue 30076 analytics ff (#30124)
Browse files Browse the repository at this point in the history
Adding some flag to turn on/off the analytics interceptor

---------

Co-authored-by: Daniel Silva <[email protected]>
  • Loading branch information
jdotcms and dsilvam authored Sep 26, 2024
1 parent 9909dff commit c7febc8
Show file tree
Hide file tree
Showing 4 changed files with 264 additions and 24 deletions.
Original file line number Diff line number Diff line change
@@ -1,50 +1,85 @@
package com.dotcms.analytics.track;

import com.dotcms.analytics.app.AnalyticsApp;
import com.dotcms.analytics.track.collectors.WebEventsCollectorServiceFactory;
import com.dotcms.analytics.track.matchers.FilesRequestMatcher;
import com.dotcms.analytics.track.matchers.PagesAndUrlMapsRequestMatcher;
import com.dotcms.analytics.track.matchers.RequestMatcher;
import com.dotcms.analytics.track.matchers.VanitiesRequestMatcher;
import com.dotcms.business.SystemTableUpdatedKeyEvent;
import com.dotcms.filters.interceptor.Result;
import com.dotcms.filters.interceptor.WebInterceptor;
import com.dotcms.security.apps.AppsAPI;
import com.dotcms.system.event.local.model.EventSubscriber;
import com.dotcms.util.CollectionsUtils;
import com.dotcms.util.WhiteBlackList;
import com.dotmarketing.beans.Host;
import com.dotmarketing.business.APILocator;
import com.dotmarketing.business.web.HostWebAPI;
import com.dotmarketing.business.web.WebAPILocator;
import com.dotmarketing.util.Config;
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.UUIDUtil;
import com.liferay.portal.model.User;
import com.liferay.util.StringPool;
import io.vavr.control.Try;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import java.util.function.Supplier;

/**
* Web Interceptor to track analytics
* @author jsanca
*/
public class AnalyticsTrackWebInterceptor implements WebInterceptor {
public class AnalyticsTrackWebInterceptor implements WebInterceptor, EventSubscriber<SystemTableUpdatedKeyEvent> {

private final static Map<String, RequestMatcher> requestMatchersMap = new ConcurrentHashMap<>();
private static final String[] DEFAULT_BLACKLISTED_PROPS = new String[]{StringPool.BLANK};
private static final String ANALYTICS_TURNED_ON_KEY = "FEATURE_FLAG_CONTENT_ANALYTICS";
private static final Map<String, RequestMatcher> requestMatchersMap = new ConcurrentHashMap<>();
private final HostWebAPI hostWebAPI;
private final AppsAPI appsAPI;
private final Supplier<User> systemUserSupplier;

/// private static final String[] DEFAULT_BLACKLISTED_PROPS = new String[]{"^/api/*"};
private static final String[] DEFAULT_BLACKLISTED_PROPS = new String[]{StringPool.BLANK};
private final WhiteBlackList whiteBlackList = new WhiteBlackList.Builder()
.addWhitePatterns(Config.getStringArrayProperty("ANALYTICS_WHITELISTED_KEYS",
new String[]{StringPool.BLANK})) // allows everything
.addBlackPatterns(CollectionsUtils.concat(Config.getStringArrayProperty( // except this
"ANALYTICS_BLACKLISTED_KEYS", new String[]{}), DEFAULT_BLACKLISTED_PROPS)).build();
private final WhiteBlackList whiteBlackList;
private final AtomicBoolean isTurnedOn;

public AnalyticsTrackWebInterceptor() {

addRequestMatcher(
this(WebAPILocator.getHostWebAPI(), APILocator.getAppsAPI(),
new WhiteBlackList.Builder()
.addWhitePatterns(Config.getStringArrayProperty("ANALYTICS_WHITELISTED_KEYS",
new String[]{StringPool.BLANK})) // allows everything
.addBlackPatterns(CollectionsUtils.concat(Config.getStringArrayProperty( // except this
"ANALYTICS_BLACKLISTED_KEYS", new String[]{}), DEFAULT_BLACKLISTED_PROPS)).build(),
new AtomicBoolean(Config.getBooleanProperty(ANALYTICS_TURNED_ON_KEY, true)),
()->APILocator.systemUser(),
new PagesAndUrlMapsRequestMatcher(),
new FilesRequestMatcher(),
// new RulesRedirectsRequestMatcher(),
// new RulesRedirectsRequestMatcher(),
new VanitiesRequestMatcher());

}

public AnalyticsTrackWebInterceptor(final HostWebAPI hostWebAPI,
final AppsAPI appsAPI,
final WhiteBlackList whiteBlackList,
final AtomicBoolean isTurnedOn,
final Supplier<User> systemUser,
final RequestMatcher... requestMatchers) {

this.hostWebAPI = hostWebAPI;
this.appsAPI = appsAPI;
this.whiteBlackList = whiteBlackList;
this.isTurnedOn = isTurnedOn;
this.systemUserSupplier = systemUser;
addRequestMatcher(requestMatchers);
}

/**
Expand All @@ -70,19 +105,59 @@ public static void removeRequestMatcher(final String requestMatcherId) {
@Override
public Result intercept(final HttpServletRequest request, final HttpServletResponse response) throws IOException {

if (whiteBlackList.isAllowed(request.getRequestURI())) {
final Optional<RequestMatcher> matcherOpt = this.anyMatcher(request, response, RequestMatcher::runBeforeRequest);
if (matcherOpt.isPresent()) {
try {
if (isAllowed(request)) {
final Optional<RequestMatcher> matcherOpt = this.anyMatcher(request, response, RequestMatcher::runBeforeRequest);
if (matcherOpt.isPresent()) {

addRequestId (request);
Logger.debug(this, () -> "intercept, Matched: " + matcherOpt.get().getId() + " request: " + request.getRequestURI());
fireNext(request, response, matcherOpt.get());
addRequestId(request);
Logger.debug(this, () -> "intercept, Matched: " + matcherOpt.get().getId() + " request: " + request.getRequestURI());
fireNext(request, response, matcherOpt.get());
}
}
} catch (Exception e) {
Logger.error(this, e.getMessage(), e);
}

return Result.NEXT;
}

/**
* If the feature flag under {@link #ANALYTICS_TURNED_ON_KEY} is on
* and there is any configuration for the analytics app
* and the white black list allowed the current request
* @param request
* @return
*/
private boolean isAllowed(final HttpServletRequest request) {

return isTurnedOn.get() &&
anyConfig(request) &&
whiteBlackList.isAllowed(request.getRequestURI());
}

private boolean anyConfig(final HttpServletRequest request) {

final Host currentSite = this.hostWebAPI.getCurrentHostNoThrow(request);

return anySecrets(currentSite);

}

/**
* Returns true if the host or the system host has any secrets for the analytics app.
* @param host
* @return
*/
private boolean anySecrets (final Host host) {

return Try.of(
() ->
this.appsAPI.getSecrets(
AnalyticsApp.ANALYTICS_APP_KEY, true, host, systemUserSupplier.get()).isPresent())
.getOrElseGet(e -> false);
}

private void addRequestId(final HttpServletRequest request) {
if (null == request.getAttribute("requestId")) {
request.setAttribute("requestId", UUIDUtil.uuid());
Expand All @@ -92,14 +167,18 @@ private void addRequestId(final HttpServletRequest request) {
@Override
public boolean afterIntercept(final HttpServletRequest request, final HttpServletResponse response) {

if (whiteBlackList.isAllowed(request.getRequestURI())) {
final Optional<RequestMatcher> matcherOpt = this.anyMatcher(request, response, RequestMatcher::runAfterRequest);
if (matcherOpt.isPresent()) {
try {
if (isAllowed(request)) {
final Optional<RequestMatcher> matcherOpt = this.anyMatcher(request, response, RequestMatcher::runAfterRequest);
if (matcherOpt.isPresent()) {

addRequestId (request);
Logger.debug(this, () -> "afterIntercept, Matched: " + matcherOpt.get().getId() + " request: " + request.getRequestURI());
fireNext(request, response, matcherOpt.get());
addRequestId(request);
Logger.debug(this, () -> "afterIntercept, Matched: " + matcherOpt.get().getId() + " request: " + request.getRequestURI());
fireNext(request, response, matcherOpt.get());
}
}
} catch (Exception e) {
Logger.error(this, e.getMessage(), e);
}

return true;
Expand Down Expand Up @@ -128,4 +207,10 @@ protected void fireNext(final HttpServletRequest request, final HttpServletRespo
}


@Override
public void notify(final SystemTableUpdatedKeyEvent event) {
if (event.getKey().contains(ANALYTICS_TURNED_ON_KEY)) {
isTurnedOn.set(Config.getBooleanProperty(ANALYTICS_TURNED_ON_KEY, true));
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.dotmarketing.filters;

import com.dotcms.analytics.track.AnalyticsTrackWebInterceptor;
import com.dotcms.business.SystemTableUpdatedKeyEvent;
import com.dotcms.ema.EMAWebInterceptor;
import com.dotcms.filters.interceptor.AbstractWebInterceptorSupportFilter;
import com.dotcms.filters.interceptor.WebInterceptorDelegate;
Expand All @@ -9,7 +10,9 @@
import com.dotcms.jitsu.EventLogWebInterceptor;
import com.dotcms.prerender.PreRenderSEOWebInterceptor;
import com.dotcms.security.multipart.MultiPartRequestSecurityWebInterceptor;
import com.dotcms.system.event.local.business.LocalSystemEventsAPI;
import com.dotcms.variant.business.web.CurrentVariantWebInterceptor;
import com.dotmarketing.business.APILocator;

import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
Expand All @@ -33,14 +36,17 @@ private void addInterceptors(final FilterConfig config) {
final WebInterceptorDelegate delegate =
this.getDelegate(config.getServletContext());

final AnalyticsTrackWebInterceptor analyticsTrackWebInterceptor = new AnalyticsTrackWebInterceptor();
delegate.add(new MultiPartRequestSecurityWebInterceptor());
delegate.add(new PreRenderSEOWebInterceptor());
delegate.add(new EMAWebInterceptor());
delegate.add(new GraphqlCacheWebInterceptor());
delegate.add(new ResponseMetaDataWebInterceptor());
delegate.add(new EventLogWebInterceptor());
delegate.add(new CurrentVariantWebInterceptor());
//delegate.add(new AnalyticsTrackWebInterceptor()); // turn on when needed.
delegate.add(analyticsTrackWebInterceptor);

APILocator.getLocalSystemEventsAPI().subscribe(SystemTableUpdatedKeyEvent.class, analyticsTrackWebInterceptor);
} // addInterceptors.

} // E:O:F:InterceptorFilter.
3 changes: 3 additions & 0 deletions dotCMS/src/main/resources/dotmarketing-config.properties
Original file line number Diff line number Diff line change
Expand Up @@ -849,6 +849,8 @@ storage.file-metadata.chain3=FILE_SYSTEM,DB,S3,MEMORY

## Telemetry
FEATURE_FLAG_TELEMETRY=false
## Analytics
FEATURE_FLAG_CONTENT_ANALYTICS=false

## Content Editor V2
CONTENT_EDITOR2_ENABLED=false
Expand All @@ -857,3 +859,4 @@ CONTENT_EDITOR2_ENABLED=false
LOCALIZATION_ENHANCEMENTS_ENABLED=false

STARTER_BUILD_VERSION=${starter.deploy.version}

Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package com.dotcms.analytics.track;

import com.dotcms.analytics.app.AnalyticsApp;
import com.dotcms.analytics.track.matchers.RequestMatcher;
import com.dotcms.security.apps.AppSecrets;
import com.dotcms.security.apps.AppsAPI;
import com.dotcms.util.WhiteBlackList;
import com.dotmarketing.beans.Host;
import com.dotmarketing.business.APILocator;
import com.dotmarketing.business.web.HostWebAPI;
import com.dotmarketing.exception.DotDataException;
import com.dotmarketing.exception.DotSecurityException;
import com.liferay.portal.model.User;
import com.liferay.util.StringPool;
import org.apache.commons.lang3.mutable.MutableBoolean;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;

/**
* This test is for the AnalyticsTrackWebInterceptor class.
* @author jsanca
*/
public class AnalyticsTrackWebInterceptorTest {

private final class TestMatcher implements RequestMatcher {

final MutableBoolean wasMatcherCalled = new MutableBoolean(false);

@Override
public boolean match(HttpServletRequest request, HttpServletResponse response) {
wasMatcherCalled.setValue(true);
return true;
}

public boolean wasCalled() {
return wasMatcherCalled.booleanValue();
}

@Override
public boolean runBeforeRequest() {
return true;
}

}

/**
* Method to test: AnalyticsTrackWebInterceptor#intercept
* Given Scenario: the feature flag is off, so the matcher test should be not called
* ExpectedResult: The test matcher should be not called
*/
@Test
public void test_intercept_feature_flag_turn_off() throws IOException {

final HostWebAPI hostWebAPI = Mockito.mock(HostWebAPI.class);
final AppsAPI appsAPI = Mockito.mock(AppsAPI.class);
final WhiteBlackList whiteBlackList = Mockito.mock(WhiteBlackList.class);
final AtomicBoolean isTurnedOn = new AtomicBoolean(false); // turn off the feature flag
final TestMatcher testMatcher = new TestMatcher();
final User user = new User();
final AnalyticsTrackWebInterceptor interceptor = new AnalyticsTrackWebInterceptor(
hostWebAPI, appsAPI, whiteBlackList, isTurnedOn, ()->user, testMatcher);
final HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
final HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
try {
interceptor.intercept(request, response);
}catch (Exception e) {}

Assert.assertFalse("The test matcher should be not called, ff is off", testMatcher.wasCalled());
}

/**
* Method to test: AnalyticsTrackWebInterceptor#intercept
* Given Scenario: the feature flag is on but not config, so the matcher test should be not called
* ExpectedResult: The test matcher should be not called
*/
@Test
public void test_intercept_feature_flag_turn_on_and_no_analytics_app() throws IOException, DotDataException, DotSecurityException {

final HostWebAPI hostWebAPI = Mockito.mock(HostWebAPI.class);
final AppsAPI appsAPI = Mockito.mock(AppsAPI.class);
final WhiteBlackList whiteBlackList = Mockito.mock(WhiteBlackList.class);
final AtomicBoolean isTurnedOn = new AtomicBoolean(true); // turn on the feature flag
final HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
final HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
final TestMatcher testMatcher = new TestMatcher();
final Host currentHost = new Host();
final User user = new User();

Mockito.when(hostWebAPI.getCurrentHostNoThrow(request)).thenReturn(currentHost);
Mockito.when(appsAPI.getSecrets(AnalyticsApp.ANALYTICS_APP_KEY,
true, currentHost, user)).thenReturn(Optional.empty()); // no config

final AnalyticsTrackWebInterceptor interceptor = new AnalyticsTrackWebInterceptor(
hostWebAPI, appsAPI, whiteBlackList, isTurnedOn, ()->user, testMatcher);

try {
interceptor.intercept(request, response);
}catch (Exception e) {}

Assert.assertFalse("The test matcher should be not called, no config set", testMatcher.wasCalled());
}

/**
* Method to test: AnalyticsTrackWebInterceptor#intercept
* Given Scenario: the feature flag is on and there is config, so the matcher test should be called
* ExpectedResult: The test matcher should be called
*/
@Test
public void test_intercept_feature_flag_turn_on_and_with_analytics_app() throws IOException, DotDataException, DotSecurityException {

final HostWebAPI hostWebAPI = Mockito.mock(HostWebAPI.class);
final AppsAPI appsAPI = Mockito.mock(AppsAPI.class);
final WhiteBlackList whiteBlackList = new WhiteBlackList.Builder()
.addWhitePatterns(new String[]{StringPool.BLANK}) // allows everything
.addBlackPatterns(new String[]{StringPool.BLANK}).build();
final AtomicBoolean isTurnedOn = new AtomicBoolean(true); // turn on the feature flag
final HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
final HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
final TestMatcher testMatcher = new TestMatcher();
final Host currentHost = new Host();
final AppSecrets appSecrets = new AppSecrets.Builder().withKey(AnalyticsApp.ANALYTICS_APP_KEY).build();
final User user = new User();

Mockito.when(hostWebAPI.getCurrentHostNoThrow(request)).thenReturn(currentHost);
Mockito.when(appsAPI.getSecrets(AnalyticsApp.ANALYTICS_APP_KEY,
true, currentHost, user)).thenReturn(Optional.of(appSecrets)); // no config
Mockito.when(request.getRequestURI()).thenReturn("/some-uri");

final AnalyticsTrackWebInterceptor interceptor = new AnalyticsTrackWebInterceptor(
hostWebAPI, appsAPI, whiteBlackList, isTurnedOn, ()->user, testMatcher);

try {
interceptor.intercept(request, response);
}catch (Exception e) {}

Assert.assertTrue("The test matcher should be called", testMatcher.wasCalled());
}

}

0 comments on commit c7febc8

Please sign in to comment.