Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update to use only CA cert #55

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,7 @@ If the TLS Certificate Authority (CA) used by Pravega is not trusted by a well-k
```
OR
```
kubectl get secret pravega-controller-tls -n nautilus-pravega -o jsonpath="{.data.tls\.crt}" | base64 --decode > ~/pravega.crt
kubectl get secret keycloak-tls -n nautilus-system -o jsonpath="{.data.tls\.crt}" | base64 --decode > ~/keycloak.crt
kubectl get secret pravega-tls -n nautilus-pravega -o jsonpath="{.data.tls\.crt}" | base64 --decode > ~/pravegaAll.crt
kubectl get cabundle sdp-default -n nautilus-system -o jsonpath="{.spec.certificates.sdp}" > ~/tlsCA-sdp.crt
```
2. On the target system, add the CA certificate to the operating system.
```shell
Expand Down
2 changes: 1 addition & 1 deletion config/checkstyle.xml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

<!-- header -->
<module name="RegexpHeader">
<property name="header" value="/\*\*\n * Copyright Pravega Authors\."/>
<property name="header" value="/\*\*\n * Copyright \(c\) Dell Inc., or its subsidiaries. All Rights Reserved\."/>
</module>
<module name="SuppressionFilter">
<property name="file" value="${suppressionsFile}"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class DeviceDriverFactory {

/**
* Instantiate a concrete subclass of DeviceDriver based on key/value properties.
* @param config
*/
DeviceDriver create(DeviceDriverConfig config) {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,19 @@
import java.util.function.Consumer;

/**
*The EventGenerator is responsible for generating events depending on file type
*The EventGenerator is responsible for generating events depending on file type.
*/
public interface EventGenerator {

/*
* Generate events from Input stream.
* Depending on file type event generation logic defers
* @param inputStream
* @param firstSequenceNumber
* @return next sequence number, end offset
* */
/**
* Generate events from Input stream.
* Depending on file type event generation logic defers
* @param inputStream
* @param firstSequenceNumber
* @param consumer
* @return next sequence number, end offset
* @throws IOException
*/
Pair<Long, Long> generateEventsFromInputStream(CountingInputStream inputStream, long firstSequenceNumber, Consumer<PravegaWriterEvent> consumer) throws IOException;

}
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ public FileIngestService(DeviceDriverConfig config) {
String getFileSpec() {
return getProperty(FILE_SPEC_KEY);
}

String getFileExtension() {
return getProperty(FILE_EXT, "");
}
Expand Down Expand Up @@ -155,6 +156,7 @@ protected void watchFiles() {
}
LOG.trace("watchFiles: END");
}

protected void processFiles() {
LOG.trace("processFiles: BEGIN");
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,11 @@ public static FileProcessor create(
* @return eventGenerator
*/
public abstract EventGenerator getEventGenerator(FileConfig config);

public void watchFiles() throws Exception {
findAndRecordNewFiles();
}

public void processFiles() throws Exception {
log.debug("processFiles: BEGIN");
if (config.enableDeleteCompletedFiles) {
Expand Down Expand Up @@ -137,7 +139,9 @@ protected void findAndRecordNewFiles() throws Exception {
}

/**
* Get directory listing.
* @return list of file name and file size in bytes
* @throws IOException If unable to list files
*/
protected List<FileNameWithOffset> getDirectoryListing() throws IOException {
log.debug("getDirectoryListing: fileSpec={}", config.fileSpec);
Expand All @@ -149,6 +153,9 @@ protected List<FileNameWithOffset> getDirectoryListing() throws IOException {
}

/**
* Get list of new files.
* @param directoryListing list of directories
* @param completedFiles list of completed files
* @return sorted list of file name and file size in bytes
*/
protected List<FileNameWithOffset> getNewFiles(List<FileNameWithOffset> directoryListing, List<FileNameWithOffset> completedFiles) {
Expand All @@ -165,7 +172,7 @@ protected List<FileNameWithOffset> getNewFiles(List<FileNameWithOffset> director
FileUtils.moveCompletedFile(dirFile, movedFilesDirectory);
log.warn("File: {} already marked as completed, moving now", dirFile.fileName);
} catch (IOException e) {
log.error("File: {} already marked as completed, but failed to move, error:{}", dirFile.fileName,e.getMessage());
log.error("File: {} already marked as completed, but failed to move, error:{}", dirFile.fileName, e.getMessage());
}
}
});
Expand Down Expand Up @@ -213,8 +220,8 @@ void processFile(FileNameWithOffset fileNameWithBeginOffset, long firstSequenceN
} catch (TxnFailedException ex) {
log.error("processFile: Write event to transaction failed with exception {} while processing file: {}, event: {}", ex, fileNameWithBeginOffset.fileName, e);

/* TODO while writing event if we get Transaction failed exception then should we abort the trasaction and process again?
This will occur only if Transaction state is not open*/
/* TODO while writing event if we get Transaction failed exception then should we abort the trasaction and process again?
This will occur only if Transaction state is not open */

throw new RuntimeException(ex);
}
Expand Down Expand Up @@ -265,18 +272,14 @@ void deleteCompletedFiles() throws Exception {
Path filePath = completedFilesPath.resolve(completedFileName);
log.debug("deleteCompletedFiles: Deleting File default name:{}, and it's completed file name:{}.", file.fileName, filePath);
try {
/**
* If file gets deleted from completed files directory, or it does not exist in default ingestion directory
* then only remove the record from DB.
*/
/* If file gets deleted from completed files directory, or it does not exist in default ingestion directory
* then only remove the record from DB. */
if (Files.deleteIfExists(filePath) || Files.notExists(Paths.get(file.fileName))) {
state.deleteCompletedFileRecord(file.fileName);
log.debug("deleteCompletedFiles: Deleted File default name:{}, and it's completed file name:{}.", file.fileName, filePath);
} else {
/**
* This situation occurs because at first attempt moving file to completed directory fails, but the file still exists in default ingestion directory.
* Moving file from default directory to completed directory will be taken care in next iteration, post which delete will be taken care.
*/
/* This situation occurs because at first attempt moving file to completed directory fails, but the file still exists in default ingestion directory.
* Moving file from default directory to completed directory will be taken care in next iteration, post which delete will be taken care. */
log.warn("deleteCompletedFiles: File {} doesn't exists in completed directory but still exist in default ingestion directory.", filePath);
}
} catch (Exception e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import java.util.function.Consumer;

/**
* Generate Event from CSV file
* Generate Event from CSV file.
*/
public class CsvFileEventGenerator implements EventGenerator {
private static final Logger LOGGER = LoggerFactory.getLogger(CsvFileEventGenerator.class);
Expand Down Expand Up @@ -66,17 +66,19 @@ public static CsvFileEventGenerator create(String routingKey, int maxRecordsPerE
/** Generate event from input stream. number of records in one event is defined in input config file
* @param inputStream
* @param firstSequenceNumber
* @param consumer
* @throws IOException If there is any error generating event.
* @return next sequence number, end offset
*/
public Pair<Long, Long> generateEventsFromInputStream(CountingInputStream inputStream, long firstSequenceNumber, Consumer<PravegaWriterEvent> consumer) throws IOException {
final CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader();
final CSVParser parser = CSVParser.parse(inputStream, StandardCharsets.UTF_8, format);
long nextSequenceNumber = firstSequenceNumber;
int numRecordsInEvent = 0;
List<HashMap<String,Object>> eventBatch = new ArrayList<>();
List<HashMap<String, Object>> eventBatch = new ArrayList<>();
for (CSVRecord record : parser) {
HashMap<String,Object> recordDataMap = new HashMap<String,Object>();
for (int i=0; i<record.size();i++) {
HashMap<String, Object> recordDataMap = new HashMap<String, Object>();
for (int i = 0; i < record.size(); i++) {
recordDataMap.put(parser.getHeaderNames().get(i), convertValue(record.get(i)));
}
eventBatch.add(recordDataMap);
Expand All @@ -101,10 +103,10 @@ public Object convertValue(String s) {
// TODO: convert timestamp
try {
return Long.parseLong(s);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException ignored) { }
try {
return Double.parseDouble(s);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException ignored) { }
return s;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public CsvFileSequenceProcessor(FileConfig config, TransactionStateDB state, Eve
}

/**
* Event generator for CSV file
* Event generator for CSV file.
* @param config configurations parameters
* @return eventGenerator
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

/**
* Copyright (c) Dell Inc., or its subsidiaries. All Rights Reserved.
*
Expand Down Expand Up @@ -48,7 +47,7 @@
import java.util.stream.Collectors;

/**
* Generate Event from Parquet file
* Generate Event from Parquet file.
*/
public class ParquetEventGenerator implements EventGenerator {
private static final Logger LOGGER = LoggerFactory.getLogger(ParquetEventGenerator.class);
Expand Down Expand Up @@ -88,6 +87,8 @@ public static ParquetEventGenerator create(String routingKey, int maxRecordsPerE
*
* @param inputStream
* @param firstSequenceNumber
* @param consumer
* @throws IOException
* @return next sequence number, end offset
*/
public Pair<Long, Long> generateEventsFromInputStream(CountingInputStream inputStream, long firstSequenceNumber, Consumer<PravegaWriterEvent> consumer) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

/**
* Copyright (c) Dell Inc., or its subsidiaries. All Rights Reserved.
*
Expand Down Expand Up @@ -28,7 +27,7 @@
import java.util.function.Consumer;

/**
* Generate Event from RAW file
* Generate Event from RAW file.
*/
public class RawEventGenerator implements EventGenerator {
private static final Logger LOGGER = LoggerFactory.getLogger(RawEventGenerator.class);
Expand Down Expand Up @@ -60,9 +59,11 @@ public static RawEventGenerator create(String routingKey) throws IOException {


/**
* Convert File to byteArray
* Convert File to byteArray.
* @param inputStream
* @param firstSequenceNumber
* @param consumer
* @throws IOException
* @return next sequence number, end offset
*/
public Pair<Long, Long> generateEventsFromInputStream(CountingInputStream inputStream, long firstSequenceNumber, Consumer<PravegaWriterEvent> consumer) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public RawFileProcessor(FileConfig config, TransactionStateDB state, EventWriter
this.writerId = writerId;
}

/** Event generator for Raw file
/** Event generator for Raw file.
* @param config configurations parameters
* @return eventGenerator
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,22 @@ public class AuthTokenDto {
public String displayName;
public String token;
public List<String> roles;
public Integer expires_in;
public Integer expiresIn;

public AuthTokenDto() {
}

public AuthTokenDto(String userId, String displayName, String token, List<String> roles, Integer expires_in) {
public AuthTokenDto(String userId, String displayName, String token, List<String> roles, Integer expiresIn) {
this.userId = userId;
this.displayName = displayName;
this.token = token;
this.roles = roles;
this.expires_in = expires_in;
this.expiresIn = expiresIn;
}

@Override
public String toString() {
return "AuthTokenDto{" + "userId=" + userId + ", displayName=" + displayName + ", token=" + token + ", roles="
+ roles + ", expires_in=" + expires_in + '}';
+ roles + ", expires_in=" + expiresIn + '}';
}
}
Loading
Loading