Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(archives): correct recording metadata in S3 tags #105

Merged
merged 2 commits into from
Nov 21, 2023
Merged

fix(archives): correct recording metadata in S3 tags #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
92b085a
fix archived recording deletion
andrewazores Aug 24, 2023
874d8ad
refactor cleanup
andrewazores Nov 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 27 additions & 9 deletions src/main/java/io/cryostat/recordings/RecordingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.Response.ResponseBuilder;
import jdk.jfr.RecordingState;
import org.apache.commons.codec.binary.Base32;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
Expand Down Expand Up @@ -122,6 +123,8 @@ public class RecordingHelper {
@Inject Clock clock;
@Inject S3Presigner presigner;

@Inject Base32 base32;

@Inject
@Named(Producers.BASE64_URL)
Base64 base64Url;
Expand Down Expand Up @@ -369,7 +372,7 @@ public String saveRecording(ActiveRecording recording, Instant expiry) throws Ex
String filename =
String.format("%s_%s_%s.jfr", transformedAlias, recording.name, timestamp);
int mib = 1024 * 1024;
String key = String.format("%s/%s", recording.target.jvmId, filename);
String key = archivedRecordingKey(recording.target.jvmId, filename);
String multipartId = null;
List<Pair<Integer, String>> parts = new ArrayList<>();
try (var stream = remoteRecordingStreamFactory.open(recording);
Expand All @@ -380,9 +383,7 @@ public String saveRecording(ActiveRecording recording, Instant expiry) throws Ex
.bucket(archiveBucket)
.key(key)
.contentType(JFR_MIME)
.tagging(
createMetadataTagging(
new Metadata(recording.metadata, expiry)));
.tagging(createActiveRecordingTagging(recording, expiry));
if (expiry != null && expiry.isAfter(Instant.now())) {
builder = builder.expires(expiry);
}
Expand Down Expand Up @@ -467,12 +468,17 @@ public String saveRecording(ActiveRecording recording, Instant expiry) throws Ex
throw e;
}
if (expiry == null) {
LinkedRecordingDescriptor serializedRecording = toExternalForm(recording);
bus.publish(
MessagingServer.class.getName(),
new Notification(
"ActiveRecordingSaved",
new RecordingEvent(
recording.target.connectUrl, toExternalForm(recording))));
new RecordingEvent(recording.target.connectUrl, serializedRecording)));
bus.publish(
MessagingServer.class.getName(),
new Notification(
"ArchivedRecordingCreated",
new RecordingEvent(recording.target.connectUrl, serializedRecording)));
}
return filename;
}
Expand All @@ -497,7 +503,11 @@ private Optional<Metadata> getArchivedRecordingMetadata(String storageKey) {
}
}

private String decodeBase64(String encoded) {
String decodeBase32(String encoded) {
return new String(base32.decode(encoded), StandardCharsets.UTF_8);
}

String decodeBase64(String encoded) {
return new String(base64Url.decode(encoded), StandardCharsets.UTF_8);
}

Expand Down Expand Up @@ -588,7 +598,7 @@ public void deleteArchivedRecording(String jvmId, String filename) {
storage.deleteObject(
DeleteObjectRequest.builder()
.bucket(archiveBucket)
.key(String.format("%s/%s", jvmId, filename))
.key(archivedRecordingKey(jvmId, filename))
.build());
bus.publish(
MessagingServer.class.getName(),
Expand All @@ -597,8 +607,16 @@ public void deleteArchivedRecording(String jvmId, String filename) {
new RecordingEvent(URI.create("localhost:0"), Map.of("name", filename))));
}

Tagging createActiveRecordingTagging(ActiveRecording recording, Instant expiry) {
Map<String, String> labels = new HashMap<>(recording.metadata.labels());
labels.put("connectUrl", recording.target.connectUrl.toString());
labels.put("jvmId", recording.target.jvmId);
Metadata metadata = new Metadata(labels, expiry);
return createMetadataTagging(metadata);
}

// Metadata
private Tagging createMetadataTagging(Metadata metadata) {
Tagging createMetadataTagging(Metadata metadata) {
// TODO attach other metadata than labels somehow. Prefixed keys to create partitioning?
var tags = new ArrayList<Tag>();
tags.addAll(
Expand Down
45 changes: 7 additions & 38 deletions src/main/java/io/cryostat/recordings/Recordings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
Expand Down Expand Up @@ -77,7 +76,6 @@
import jakarta.ws.rs.core.HttpHeaders;
import jakarta.ws.rs.core.Response;
import jdk.jfr.RecordingState;
import org.apache.commons.codec.binary.Base32;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
Expand All @@ -99,8 +97,6 @@
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.model.Tag;
import software.amazon.awssdk.services.s3.model.Tagging;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
Expand All @@ -121,8 +117,6 @@ public class Recordings {
@Inject ObjectMapper mapper;
@Inject RecordingHelper recordingHelper;

@Inject Base32 base32;

@Inject
@Named(Producers.BASE64_URL)
Base64 base64Url;
Expand Down Expand Up @@ -220,6 +214,7 @@ public void agentPush(
if (rawLabels != null) {
rawLabels.getMap().forEach((k, v) -> labels.put(k, v.toString()));
}
labels.put("jvmId", jvmId);
Metadata metadata = new Metadata(labels);
logger.infov(
"recording:{0}, labels:{1}, maxFiles:{2}", recording.fileName(), labels, maxFiles);
Expand Down Expand Up @@ -345,13 +340,15 @@ Map<String, Object> doUpload(FileUpload recording, Metadata metadata, String jvm
if (!filename.endsWith(".jfr")) {
filename = filename + ".jfr";
}
Map<String, String> labels = new HashMap<>(metadata.labels);
labels.put("jvmId", jvmId);
String key = recordingHelper.archivedRecordingKey(jvmId, filename);
storage.putObject(
PutObjectRequest.builder()
.bucket(archiveBucket)
.key(key)
.contentType(RecordingHelper.JFR_MIME)
.tagging(createMetadataTagging(metadata))
.tagging(recordingHelper.createMetadataTagging(new Metadata(labels)))
.build(),
RequestBody.fromFile(recording.filePath()));
logger.info("Upload complete");
Expand Down Expand Up @@ -663,7 +660,7 @@ public void deleteRecording(@RestPath long targetId, @RestPath long remoteId) th
@RolesAllowed("write")
public void deleteArchivedRecording(@RestPath String encodedJvmId, @RestPath String filename)
throws Exception {
var jvmId = decodeBase32(encodedJvmId);
var jvmId = recordingHelper.decodeBase32(encodedJvmId);
logger.infov("Handling archived recording deletion: {0} / {1}", jvmId, filename);
var metadata =
recordingHelper
Expand All @@ -676,7 +673,8 @@ public void deleteArchivedRecording(@RestPath String encodedJvmId, @RestPath Str
.map(c -> c.toString())
.orElseGet(() -> metadata.labels.computeIfAbsent("connectUrl", k -> jvmId));
logger.infov(
"Archived recording from connectUrl {0} has metadata: {1}", connectUrl, metadata);
"Archived recording from connectUrl \"{0}\" has metadata: {1}",
connectUrl, metadata);
logger.infov(
"Sending S3 deletion request for {0} {1}",
archiveBucket, recordingHelper.archivedRecordingKey(jvmId, filename));
Expand Down Expand Up @@ -834,35 +832,6 @@ public Response redirectPresignedDownload(@RestPath String encodedKey)
.build();
}

private Tagging createMetadataTagging(Metadata metadata) {
// TODO attach other metadata than labels somehow. Prefixed keys to create partitioning?
return Tagging.builder()
.tagSet(
metadata.labels.entrySet().stream()
.map(
e ->
Tag.builder()
.key(
base64Url.encodeAsString(
e.getKey()
.getBytes(
StandardCharsets
.UTF_8)))
.value(
base64Url.encodeAsString(
e.getValue()
.getBytes(
StandardCharsets
.UTF_8)))
.build())
.toList())
.build();
}

private String decodeBase32(String encoded) {
return new String(base32.decode(encoded), StandardCharsets.UTF_8);
}

private static Map<String, Object> getRecordingOptions(
IFlightRecorderService service, RecordingOptionsBuilder builder) throws Exception {
IConstrainedMap<String> recordingOptions = builder.build();
Expand Down
Loading