Skip to content

Commit

Permalink
[MRESOLVER-587] Memory usage improvements (#537)
Browse files Browse the repository at this point in the history 
Do not use `Object` as descriptor key, use dedicated type. And intern the `List<Dependency>` on artifact descriptors. Also introduce "intern"-ing of ArtifactDescriptor dependencies and managedDependencies that is configurable (speed vs memory).

---

https://issues.apache.org/jira/browse/MRESOLVER-587
  • Loading branch information
@cstamas
cstamas authored Aug 2, 2024
1 parent 7bfa410 commit e48e42a
Show file tree
Hide file tree
Showing 4 changed files with 210 additions and 101 deletions.
124 changes: 115 additions & 9 deletions maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,49 @@ public final class DataPool {
*/
public static final String CONFIG_PROP_COLLECTOR_POOL_DESCRIPTOR = CONFIG_PROPS_PREFIX + "descriptor";

/**
* Flag controlling interning data pool type used by dependency lists collector for ArtifactDescriptor (POM) instances,
* matters for heap consumption. By default, uses “weak” references (consume less heap). Using “hard” will make it
* much more memory aggressive and possibly faster (system and Java dependent). Supported values: "hard", "weak".
*
* @since 1.9.22
* @configurationSource {@link RepositorySystemSession#getConfigProperties()}
* @configurationType {@link java.lang.String}
* @configurationDefaultValue {@link #HARD}
*/
public static final String CONFIG_PROP_COLLECTOR_POOL_DEPENDENCY_LISTS =
"aether.dependencyCollector.pool.dependencyLists";

/**
* Flag controlling interning artifact descriptor dependencies.
*
* @since 1.9.22
* @configurationSource {@link RepositorySystemSession#getConfigProperties()}
* @configurationType {@link java.lang.Boolean}
* @configurationDefaultValue false
*/
public static final String CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_DEPENDENCIES =
"aether.dependencyCollector.pool.internArtifactDescriptorDependencies";

/**
* Flag controlling interning artifact descriptor managed dependencies.
*
* @since 1.9.22
* @configurationSource {@link RepositorySystemSession#getConfigProperties()}
* @configurationType {@link java.lang.Boolean}
* @configurationDefaultValue true
*/
public static final String CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_MANAGED_DEPENDENCIES =
"aether.dependencyCollector.pool.internArtifactDescriptorManagedDependencies";

private static final String ARTIFACT_POOL = DataPool.class.getName() + "$Artifact";

private static final String DEPENDENCY_POOL = DataPool.class.getName() + "$Dependency";

private static final String DESCRIPTORS = DataPool.class.getName() + "$Descriptors";

private static final String DEPENDENCY_LISTS_POOL = DataPool.class.getName() + "$DependencyLists";

public static final ArtifactDescriptorResult NO_DESCRIPTOR =
new ArtifactDescriptorResult(new ArtifactDescriptorRequest());

Expand All @@ -112,7 +149,12 @@ public final class DataPool {
/**
* Descriptor interning pool, lives across session (if session carries non-null {@link RepositoryCache}).
*/
private final InternPool<Object, Descriptor> descriptors;
private final InternPool<DescriptorKey, Descriptor> descriptors;

/**
* {@link Dependency} list interning pool, lives across session (if session carries non-null {@link RepositoryCache}).
*/
private final InternPool<List<Dependency>, List<Dependency>> dependencyLists;

/**
* Constraint cache, lives during single collection invocation (same as this DataPool instance).
Expand All @@ -124,17 +166,29 @@ public final class DataPool {
*/
private final ConcurrentHashMap<Object, List<DependencyNode>> nodes;

private final boolean internArtifactDescriptorDependencies;

private final boolean internArtifactDescriptorManagedDependencies;

@SuppressWarnings("unchecked")
public DataPool(RepositorySystemSession session) {
final RepositoryCache cache = session.getCache();

internArtifactDescriptorDependencies = ConfigUtils.getBoolean(
session, false, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_DEPENDENCIES);
internArtifactDescriptorManagedDependencies = ConfigUtils.getBoolean(
session, true, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_MANAGED_DEPENDENCIES);

InternPool<Artifact, Artifact> artifactsPool = null;
InternPool<Dependency, Dependency> dependenciesPool = null;
InternPool<Object, Descriptor> descriptorsPool = null;
InternPool<DescriptorKey, Descriptor> descriptorsPool = null;
InternPool<List<Dependency>, List<Dependency>> dependencyListsPool = null;
if (cache != null) {
artifactsPool = (InternPool<Artifact, Artifact>) cache.get(session, ARTIFACT_POOL);
dependenciesPool = (InternPool<Dependency, Dependency>) cache.get(session, DEPENDENCY_POOL);
descriptorsPool = (InternPool<Object, Descriptor>) cache.get(session, DESCRIPTORS);
descriptorsPool = (InternPool<DescriptorKey, Descriptor>) cache.get(session, DESCRIPTORS);
dependencyListsPool =
(InternPool<List<Dependency>, List<Dependency>>) cache.get(session, DEPENDENCY_LISTS_POOL);
}

if (artifactsPool == null) {
Expand Down Expand Up @@ -164,9 +218,20 @@ public DataPool(RepositorySystemSession session) {
}
}

if (dependencyListsPool == null) {
String dependencyListsPoolType =
ConfigUtils.getString(session, HARD, CONFIG_PROP_COLLECTOR_POOL_DEPENDENCY_LISTS);

dependencyListsPool = createPool(dependencyListsPoolType);
if (cache != null) {
cache.put(session, DEPENDENCY_LISTS_POOL, dependencyListsPool);
}
}

this.artifacts = artifactsPool;
this.dependencies = dependenciesPool;
this.descriptors = descriptorsPool;
this.dependencyLists = dependencyListsPool;

this.constraints = new ConcurrentHashMap<>(256);
this.nodes = new ConcurrentHashMap<>(256);
Expand All @@ -180,26 +245,36 @@ public Dependency intern(Dependency dependency) {
return dependencies.intern(dependency, dependency);
}

public Object toKey(ArtifactDescriptorRequest request) {
return request.getArtifact();
public DescriptorKey toKey(ArtifactDescriptorRequest request) {
return new DescriptorKey(request.getArtifact());
}

public ArtifactDescriptorResult getDescriptor(Object key, ArtifactDescriptorRequest request) {
public ArtifactDescriptorResult getDescriptor(DescriptorKey key, ArtifactDescriptorRequest request) {
Descriptor descriptor = descriptors.get(key);
if (descriptor != null) {
return descriptor.toResult(request);
}
return null;
}

public void putDescriptor(Object key, ArtifactDescriptorResult result) {
public void putDescriptor(DescriptorKey key, ArtifactDescriptorResult result) {
if (internArtifactDescriptorDependencies) {
result.setDependencies(intern(result.getDependencies()));
}
if (internArtifactDescriptorManagedDependencies) {
result.setManagedDependencies(intern(result.getManagedDependencies()));
}
descriptors.intern(key, new GoodDescriptor(result));
}

public void putDescriptor(Object key, ArtifactDescriptorException e) {
public void putDescriptor(DescriptorKey key, ArtifactDescriptorException e) {
descriptors.intern(key, BadDescriptor.INSTANCE);
}

private List<Dependency> intern(List<Dependency> dependencies) {
return dependencyLists.intern(dependencies, dependencies);
}

public Object toKey(VersionRangeRequest request) {
return new ConstraintKey(request);
}
Expand Down Expand Up @@ -234,8 +309,39 @@ public void putChildren(Object key, List<DependencyNode> children) {
nodes.put(key, children);
}

abstract static class Descriptor {
public static final class DescriptorKey {
private final Artifact artifact;
private final int hashCode;

private DescriptorKey(Artifact artifact) {
this.artifact = artifact;
this.hashCode = Objects.hashCode(artifact);
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
DescriptorKey that = (DescriptorKey) o;
return Objects.equals(artifact, that.artifact);
}

@Override
public int hashCode() {
return hashCode;
}

@Override
public String toString() {
return getClass().getSimpleName() + "{" + "artifact='" + artifact + '\'' + '}';
}
}

abstract static class Descriptor {
public abstract ArtifactDescriptorResult toResult(ArtifactDescriptorRequest request);
}

Expand Down
2 changes: 1 addition & 1 deletion ...l/src/main/java/org/eclipse/aether/internal/impl/collect/DependencyCollectorDelegate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ protected ArtifactDescriptorResult resolveCachedArtifactDescriptor(
Dependency d,
Results results,
List<DependencyNode> nodes) {
Object key = pool.toKey(descriptorRequest);
DataPool.DescriptorKey key = pool.toKey(descriptorRequest);
ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest);
if (descriptorResult == null) {
try {
Expand Down
2 changes: 1 addition & 1 deletion maven-resolver-impl/src/test/java/org/eclipse/aether/internal/impl/collect/DataPoolTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ void testArtifactDescriptorCaching() {
result.addAlias(new DefaultArtifact("gid:alias:4"));

DataPool pool = newDataPool();
Object key = pool.toKey(request);
DataPool.DescriptorKey key = pool.toKey(request);
pool.putDescriptor(key, result);
ArtifactDescriptorResult cached = pool.getDescriptor(key, request);
assertNotNull(cached);
Expand Down
Loading

0 comments on commit e48e42a

Please sign in to comment.