Skip to content

Commit

Permalink
Merge pull request #2136 from openturing/sprinklr/asset
Browse files Browse the repository at this point in the history 
Sprinklr/asset
  • Loading branch information
@alegauss
alegauss authored Sep 19, 2024
2 parents 60f0ba0 + 6b79a53 commit ab6f28e
Show file tree
Hide file tree
Showing 7 changed files with 370 additions and 40 deletions.
6 changes: 6 additions & 0 deletions turing-commons/src/main/java/com/viglet/turing/commons/cache/TurCustomClassCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,16 @@ private TurCustomClassCache() {
}
private static final Map<String, Object> customClassMap = new HashMap<>();

/**
* Retrieve an instance of a class from a static HashMap. The class will be instantiated on the HashMap if not present.
* @param className The name of the class that has to be retrieved.
* @return An Optional of the instance.
*/
public static Optional<Object> getCustomClassMap(String className) {
if (!customClassMap.containsKey(className)) {
log.info("Custom class {} not found in memory, instancing...", className);
try {
// Instância a classe dinamicamente, sem precisar definir o import no momento de compilação
customClassMap.put(className, Objects.requireNonNull(Class.forName(className)
.getDeclaredConstructor().newInstance()));
} catch (InstantiationException | IllegalAccessException | InvocationTargetException |
Expand Down
192 changes: 152 additions & 40 deletions ...r/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrProcess.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
import com.viglet.turing.connector.sprinklr.persistence.model.TurSprinklrAttributeMapping;
import com.viglet.turing.connector.sprinklr.persistence.model.TurSprinklrSource;
import com.viglet.turing.connector.sprinklr.persistence.repository.TurSprinklrAttributeMappingRepository;
import com.viglet.turing.connector.sprinklr.utils.FileAsset;
import com.viglet.turing.connector.sprinklr.utils.FileAssetsExtractor;
import com.viglet.turing.sprinklr.client.service.kb.TurSprinklrKBService;
import com.viglet.turing.sprinklr.client.service.kb.response.TurSprinklrKBSearch;
import com.viglet.turing.sprinklr.client.service.kb.response.TurSprinklrSearchResult;
Expand All @@ -25,7 +27,7 @@
import com.viglet.turing.sprinklr.client.service.token.TurSprinklrTokenService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.annotation.Value; // Gets the value from application.properties
import org.springframework.stereotype.Component;

import java.net.MalformedURLException;
Expand All @@ -38,9 +40,12 @@
public class TurSprinklrProcess {
private final String turingUrl;
private final String turingApiKey;
private TurSNJobItems turSNJobItems = new TurSNJobItems();
private final int jobSize;
private final TurSprinklrAttributeMappingRepository turSprinklrAttributeMappingRepository;
/**
* Represents the jobs that will be sent to Turing API
*/
private TurSNJobItems turSNJobItems = new TurSNJobItems();

@Inject
public TurSprinklrProcess(@Value("${turing.url}") String turingUrl,
Expand All @@ -53,41 +58,116 @@ public TurSprinklrProcess(@Value("${turing.url}") String turingUrl,
this.turSprinklrAttributeMappingRepository = turSprinklrAttributeMappingRepository;
}

private static void addItemInExistingAttribute(String attributeValue,
Map<String, Object> attributes,
String attributeName) {
if (attributes.get(attributeName) instanceof ArrayList)
addItemToArray(attributes, attributeName, attributeValue);
else convertAttributeSingleValueToArray(attributes, attributeName, attributeValue);
}

private static void addItemToArray(Map<String, Object> attributes, String attributeName, String attributeValue) {
List<String> attributeValues = new ArrayList<>(((List<?>) attributes.get(attributeName))
.stream().map(String.class::cast).toList());
attributeValues.add(attributeValue);
attributes.put(attributeName, attributeValues);

}

private static void convertAttributeSingleValueToArray(Map<String, Object> attributes,
String attributeName, String attributeValue) {
List<Object> attributeValues = new ArrayList<>();
attributeValues.add(attributes.get(attributeName));
attributeValues.add(attributeValue);
attributes.put(attributeName, attributeValues);
}

public void start(TurSprinklrSource turSprinklrSource) {
reset();
// Index for the pagination parameter of the Knowledge Base API, it starts on 0
AtomicInteger kbPage = new AtomicInteger(0);

// Get a token for the API
TurSprinklrTokenService turSprinklrTokenService = new TurSprinklrTokenService(
TurSprinklrSecretKey.builder()
.apiKey(turSprinklrSource.getApiKey())
.secretKey(turSprinklrSource.getSecretKey())
.environment(turSprinklrSource.getEnvironment())
.build());
TurSprinklrAccessToken turSprinklrAccessToken = turSprinklrTokenService
.getAccessToken();
.build()
);
TurSprinklrAccessToken turSprinklrAccessToken = turSprinklrTokenService.getAccessToken();

if (turSprinklrAccessToken != null) {
while (true) {
TurSprinklrKBSearch turSprinklrKBSearch = TurSprinklrKBService.run(turSprinklrAccessToken, kbPage.get());
if (!(turSprinklrKBSearch == null)) {

if (turSprinklrKBSearch != null) {
List<TurSprinklrSearchResult> results = turSprinklrKBSearch.getData().getSearchResults();

if (results.isEmpty()) {
break;
} else {
results.forEach(searchResult -> {
getPage(turSprinklrSource, searchResult, turSprinklrAccessToken);
Locale resultLocale = searchResult.getLocale();
Collection<String> turSites = turSprinklrSource.getTurSNSites();

// Inserts new jobs into turSNJobItems
getArticle(turSprinklrSource, searchResult, turSprinklrAccessToken);

// Gets the assets attached to the search result and inserts into turSNJobItems.
List<FileAsset> assets = getFileAssets(searchResult);
addFileAssetsToJobItens(assets, resultLocale, turSites);

// Quando o tamanho de turSNJobItems alcançar o JobSize definido, envia para o turing.
sendToTuringWhenMaxSize();

getInfoQueue();
});
// Increment Index
kbPage.incrementAndGet();
}
}
}
}
if (turSNJobItems.size() > 0) {
// Envia os últimos jobs restantes.
sendToTuring();
getInfoQueue();
}
}

/**
* Extracts the file assets from the search result and returns a list of FileAsset objects.
*/
private List<FileAsset> getFileAssets(TurSprinklrSearchResult searchResult) {
final var fileAssetExtractor = new FileAssetsExtractor(turingUrl, turingApiKey);
final var fileAssets = fileAssetExtractor.extractFromLinkedAssets(searchResult);

if (fileAssets == null || fileAssets.isEmpty()) {
return Collections.emptyList();
}
return fileAssets;
}

/**
* Adds the file assets to the job items list.
*/
private void addFileAssetsToJobItens(List<FileAsset> fileAssets, Locale locale, Collection<String> turSites) {
for (var asset : fileAssets) {
var turSNJobItemAttributes = asset.toMapAttributes();
TurSNJobItem turSNJobItem = new TurSNJobItem(
TurSNJobAction.CREATE,
(List<String>) turSites,
locale,
turSNJobItemAttributes
);
turSNJobItems.add(turSNJobItem);
}
}

/**
* Clears the List of jobs in Turing API
*/
private void reset() {
turSNJobItems = new TurSNJobItems();
}
Expand All @@ -96,13 +176,20 @@ private void getInfoQueue() {
log.info("Total Job Item: {}", Iterators.size(turSNJobItems.iterator()));
}

public void getPage(TurSprinklrSource turSprinklrSource, TurSprinklrSearchResult searchResult,
TurSprinklrAccessToken token) {
public void getArticle(TurSprinklrSource turSprinklrSource, TurSprinklrSearchResult searchResult,
TurSprinklrAccessToken token) {
log.info("{}: {}", searchResult.getId(), turSprinklrSource.getTurSNSites());
addTurSNJobItems(turSprinklrSource, searchResult, token);

}

/**
* Inserts a job in the job list parameter of this class (turSNJobItems)
*
* @param turSprinklrSource Source to extract the Semantic Navigation sites, locale and attributes
* @param searchResult Source to extract Locale and attributes
* @param token N/A
*/
private void addTurSNJobItems(TurSprinklrSource turSprinklrSource, TurSprinklrSearchResult searchResult,
TurSprinklrAccessToken token) {
TurSNJobItem turSNJobItem = new TurSNJobItem(TurSNJobAction.CREATE, new ArrayList<>(turSprinklrSource.getTurSNSites()),
Expand All @@ -111,8 +198,23 @@ private void addTurSNJobItems(TurSprinklrSource turSprinklrSource, TurSprinklrSe
turSNJobItems.add(turSNJobItem);
}

/**
* Get the Locale from inside turSprinklrSource entity, if it does not work, tries to get the locale from inside
* locale_class column of turSprinklrSource entity, if also does not work, return Locale.US.
*
* @param turSprinklrSource the method will first try to get the Locale from turSprinklrSource.getLocale().
* @param searchResult is used to create a context where an attempt will be made to retrieve the locale.
* @param token N/A
* @return Locale based on the parameters.
*/
public Locale getLocale(TurSprinklrSource turSprinklrSource, TurSprinklrSearchResult searchResult,
TurSprinklrAccessToken token) {
/*
Try to get extract the locale from turSprinklrSource.getLocale()
Or else extracts the locale class (by default sprinklr-commons TurSprinklrExtLocal) from turSprinklrSource.
If getCustomClassMap found a class, converts the result to an instance of TurSprinklrExtLocaleInterface.
Then calls the .consume method to get a Locale, if none of this works, then return Locale.US
*/
return Optional.ofNullable(turSprinklrSource.getLocale())
.orElseGet(() -> {
if (!StringUtils.isEmpty(turSprinklrSource.getLocaleClass())) {
Expand All @@ -131,22 +233,48 @@ private void sendToTuringWhenMaxSize() {
}
}

/**
* Returns a new HashMap of "Attribute(turing Field, Attribute Name from export.json) -> Attribute Value". <p>
* This method is used when sending a job to Turing.
*
* @param turSprinklrSource Is used to find the <b>turSprinklrAttributeMapping entity</b>, it represents
* <code>export.json</code> file.
* @param token N/A
* @param searchResult If a <b>CustomClass</b> is defined by <code>export.json</code> file, the value will be extracted from <b>searchResult</b>
* @return the created HashMap
*/
public Map<String, Object> getJobItemAttributes(TurSprinklrSource turSprinklrSource, TurSprinklrAccessToken token,
TurSprinklrSearchResult searchResult) {
Map<String, Object> turSNJobItemAttributes = new HashMap<>();
turSprinklrAttributeMappingRepository.findByTurSprinklrSource(turSprinklrSource).ifPresent(source -> source.forEach(turSprinklrCustomClass ->
Optional.ofNullable(turSprinklrCustomClass.getText()).ifPresentOrElse(text ->
turSNJobItemAttributes.put(turSprinklrCustomClass.getName(), text)
/* Example
NAME | TEXT(VALUE)
source_apps | SPRINKLR (Hardcoded)
created_date | 19-02-2015 (retrieved using a custom class)
*/

/*
Will retrieve the objects from 'turSprinklrAttributeMapping' entity, then for each one of the objects that this entity contains. Will get
the 'text' attribute of this object.
If the 'text' attribute is present, then will create a new key in turSNJobItemAttributes with the `name` attribute of this same object. The
value of this key will be the 'text'.
If not present, tries to get the ClassName from `AttributeMapping`, instantiate it dynamically and consumes it to get the value of the key.
*/
turSprinklrAttributeMappingRepository.findByTurSprinklrSource(turSprinklrSource).ifPresent(mapping -> mapping.forEach(attribute ->
Optional.ofNullable(attribute.getText()).ifPresentOrElse(text ->
turSNJobItemAttributes.put(attribute.getName(), text)
, () -> {
if (!StringUtils.isEmpty(turSprinklrCustomClass.getClassName()))
getCustomClass(searchResult, token, turSprinklrCustomClass)
// Se o campo ClassName estiver preenchido no Export.json
if (!StringUtils.isEmpty(attribute.getClassName()))
getCustomClass(searchResult, token, attribute)
.ifPresent(turMultiValue -> turMultiValue.forEach(attributeValue -> {
if (!StringUtils.isBlank(attributeValue)) {
if (turSNJobItemAttributes.containsKey(turSprinklrCustomClass.getName())) {
if (turSNJobItemAttributes.containsKey(attribute.getName())) {
addItemInExistingAttribute(attributeValue,
turSNJobItemAttributes, turSprinklrCustomClass.getName());
turSNJobItemAttributes, attribute.getName());
} else {
addFirstItemToAttribute(turSprinklrCustomClass.getName(),
addFirstItemToAttribute(attribute.getName(),
attributeValue, turSNJobItemAttributes);
}
}
Expand All @@ -163,43 +291,27 @@ private Optional<TurMultiValue> getCustomClass(TurSprinklrSearchResult searchRes
.consume(getTurSprinklrContext(searchResult, token)));
}

/**
* Builds a TurSprinklrContext object from a searchResult and a token object
*/
public TurSprinklrContext getTurSprinklrContext(TurSprinklrSearchResult searchResult, TurSprinklrAccessToken token) {
return TurSprinklrContext.builder()
.searchResult(searchResult)
.accessToken(token)
.build();
}

// Used only for getJobItemAttributes
private void addFirstItemToAttribute(String attributeName,
String attributeValue,
Map<String, Object> attributes) {
attributes.put(attributeName, attributeValue);
}

private static void addItemInExistingAttribute(String attributeValue,
Map<String, Object> attributes,
String attributeName) {
if (attributes.get(attributeName) instanceof ArrayList)
addItemToArray(attributes, attributeName, attributeValue);
else convertAttributeSingleValueToArray(attributes, attributeName, attributeValue);
}

private static void addItemToArray(Map<String, Object> attributes, String attributeName, String attributeValue) {
List<String> attributeValues = new ArrayList<>(((List<?>) attributes.get(attributeName))
.stream().map(String.class::cast).toList());
attributeValues.add(attributeValue);
attributes.put(attributeName, attributeValues);

}

private static void convertAttributeSingleValueToArray(Map<String, Object> attributes,
String attributeName, String attributeValue) {
List<Object> attributeValues = new ArrayList<>();
attributeValues.add(attributes.get(attributeName));
attributeValues.add(attributeValue);
attributes.put(attributeName, attributeValues);
}

/**
* Push current turSNJobItems to turing.
*/
private void sendToTuring() {
if (log.isDebugEnabled()) {
for (TurSNJobItem turSNJobItem : turSNJobItems) {
Expand Down
3 changes: 3 additions & 0 deletions ...c/main/java/com/viglet/turing/connector/sprinklr/persistence/model/TurSprinklrSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
import java.util.HashSet;
import java.util.Locale;

/**
* An entity that represents the connections to Sprinklr servers
*/
@Builder(toBuilder = true)
@AllArgsConstructor
@NoArgsConstructor
Expand Down
56 changes: 56 additions & 0 deletions ...nklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/utils/FileAsset.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package com.viglet.turing.connector.sprinklr.utils;

import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;

import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* Represents a File Asset extracted from Sprinklr Knowledge Base search result.
* @author Gabriel F. Gomazako
* @since 0.3.9
*/
@AllArgsConstructor
@NoArgsConstructor
public class FileAsset {
private String id;
private String filename;
private String ocrContent;
private Date indexingDate;
private Date modificationDate;
private URL url;
private long fileSize;
private String extension;
private String assetType;
private String assetCategory;

/**
* Converts this FileAsset to an attribute map.
* @return the attribute map.
*/
public Map<String, Object> toMapAttributes() {
var attributes = new HashMap<String, Object>();

var formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
String formatedIndexingDate = formatter.format(indexingDate);
String formatedModificationDate = formatter.format(modificationDate);

attributes.put("id", id);
attributes.put("title", filename);
attributes.put("text", ocrContent);
attributes.put("publication_date", formatedIndexingDate);
attributes.put("modification_date", formatedModificationDate);
attributes.put("url", url);
attributes.put("filesize", fileSize);
attributes.put("extension", extension);
attributes.put("source_apps", List.of("SPRINKLR"));
attributes.put("type", "Static File");

return attributes;
}
}
Loading

0 comments on commit ab6f28e

Please sign in to comment.