WIP adding security resource sharing SPI and sample-extension-plugin

build.gradle

@@ -569,6 +569,7 @@ tasks.integrationTest.finalizedBy(jacocoTestReport) // report is always generate
 check.dependsOn integrationTest
 
 dependencies {
+    implementation project(path: ":${rootProject.name}-spi", configuration: 'shadow')
     implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
     implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}"
     implementation "org.apache.httpcomponents.client5:httpclient5-cache:${versions.httpclient5}"

sample-extension-plugin/build.gradle

@@ -0,0 +1,188 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+apply plugin: 'opensearch.opensearchplugin'
+apply plugin: 'opensearch.testclusters'
+apply plugin: 'opensearch.java-rest-test'
+
+import org.opensearch.gradle.test.RestIntegTestTask
+import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
+import org.apache.tools.ant.taskdefs.condition.Os
+
+import java.util.concurrent.Callable
+
+
+opensearchplugin {
+    name 'opensearch-security-sample-extension'
+    description 'Sample plugin that extends OpenSearch Security Resource Sharing plugin'
+    classname 'org.opensearch.security.sampleextension.SampleExtensionPlugin'
+    extendedPlugins = ['opensearch-security']
+}
+
+ext {
+    projectSubstitutions = [:]
+    licenseFile = rootProject.file('LICENSE.txt')
+    noticeFile = rootProject.file('NOTICE.txt')
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
+}
+
+dependencies {
+    compileOnly project(path: ":${rootProject.name}-spi", configuration: 'shadow')
+}
+
+def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
+es_tmp_dir.mkdirs()
+
+File repo = file("$buildDir/testclusters/repo")
+def _numNodes = findProperty('numNodes') as Integer ?: 1
+
+licenseHeaders.enabled = true
+validateNebulaPom.enabled = false
+testingConventions.enabled = false
+loggerUsageCheck.enabled = false
+
+javaRestTest.dependsOn(rootProject.assemble)
+javaRestTest {
+    systemProperty 'tests.security.manager', 'false'
+}
+testClusters.javaRestTest {
+    testDistribution = 'INTEG_TEST'
+}
+
+task integTest(type: RestIntegTestTask) {
+    description = "Run tests against a cluster"
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+}
+tasks.named("check").configure { dependsOn(integTest) }
+
+integTest {
+    if (project.hasProperty('excludeTests')) {
+        project.properties['excludeTests']?.replaceAll('\\s', '')?.split('[,;]')?.each {
+            exclude "${it}"
+        }
+    }
+    systemProperty 'tests.security.manager', 'false'
+    systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath
+
+    systemProperty "https", System.getProperty("https")
+    systemProperty "user", System.getProperty("user")
+    systemProperty "password", System.getProperty("password")
+    // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
+    // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
+    doFirst {
+        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
+        // use longer timeouts for requests.
+        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
+        systemProperty 'cluster.debug', isDebuggingCluster
+        // Set number of nodes system property to be used in tests
+        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
+        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
+        // not being written, the waitForAllConditions ensures it's written
+        getClusters().forEach { cluster ->
+            cluster.waitForAllConditions()
+        }
+    }
+
+    // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable
+    if (System.getProperty("test.debug") != null) {
+        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000'
+    }
+    if (System.getProperty("tests.rest.bwcsuite") == null) {
+        filter {
+            excludeTestsMatching "org.opensearch.security.sampleextension.bwc.*IT"
+        }
+    }
+}
+project.getTasks().getByName('bundlePlugin').dependsOn(rootProject.tasks.getByName('build'))
+Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
+Zip rootBundle = (Zip) rootProject.getTasks().getByName("bundlePlugin");
+integTest.dependsOn(bundle)
+integTest.getClusters().forEach{c -> {
+    c.plugin(rootProject.getObjects().fileProperty().value(rootBundle.getArchiveFile()))
+    c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))
+}}
+
+testClusters.integTest {
+    testDistribution = 'INTEG_TEST'
+
+    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
+    if (_numNodes > 1) numberOfNodes = _numNodes
+    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
+    // i.e. we have to use a custom property to flag when we want to debug OpenSearch JVM
+    // since we also support multi node integration tests we increase debugPort per node
+    if (System.getProperty("cluster.debug") != null) {
+        def debugPort = 5005
+        nodes.forEach { node ->
+            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
+            debugPort += 1
+        }
+    }
+    setting 'path.repo', repo.absolutePath
+}
+
+afterEvaluate {
+    testClusters.integTest.nodes.each { node ->
+        def plugins = node.plugins
+        def firstPlugin = plugins.get(0)
+        if (firstPlugin.provider == project.bundlePlugin.archiveFile) {
+            plugins.remove(0)
+            plugins.add(firstPlugin)
+        }
+
+        node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem"))
+        node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem"))
+        node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem"))
+        node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem"))
+        node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem"))
+        node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+        node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+        node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+        node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+        node.setting("plugins.security.ssl.http.enabled", "true")
+        node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+        node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+        node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+        node.setting("plugins.security.allow_unsafe_democertificates", "true")
+        node.setting("plugins.security.allow_default_init_securityindex", "true")
+        node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de")
+        node.setting("plugins.security.audit.type", "internal_opensearch")
+        node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
+        node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
+        node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
+    }
+}
+
+run {
+    doFirst {
+        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
+        // not being written, the waitForAllConditions ensures it's written
+        getClusters().forEach { cluster ->
+            cluster.waitForAllConditions()
+        }
+    }
+    useCluster testClusters.integTest
+}
+
+// As of ES 7.7 the sample-extension-plugin is being added to the list of plugins for the testCluster during build before
+// the security plugin is causing build failures.
+// The security zip is added explicitly above but the sample-extension-plugin is added implicitly at some time during evaluation.
+// Will need to do a deep dive to find out exactly what task adds the sample-extension-plugin and add security there but a temporary hack is to
+// reorder the plugins list after evaluation but prior to task execution when the plugins are installed.
+//afterEvaluate {
+//    testClusters.javaRestTest.nodes.each { node ->
+//        def nodePlugins = node.plugins
+//        def firstPlugin = nodePlugins.get(0)
+//        if (firstPlugin.provider == project.bundlePlugin.archiveFile) {
+//            nodePlugins.remove(0)
+//            nodePlugins.add(firstPlugin)
+//        }
+//    }
+//}

sample-extension-plugin/src/main/java/org/opensearch/security/sampleextension/SampleExtensionPlugin.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.security.sampleextension;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import org.opensearch.action.ActionRequest;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.IndexScopedSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.SettingsFilter;
+import org.opensearch.core.action.ActionResponse;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.env.Environment;
+import org.opensearch.env.NodeEnvironment;
+import org.opensearch.indices.SystemIndexDescriptor;
+import org.opensearch.plugins.ActionPlugin;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.SystemIndexPlugin;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.rest.RestController;
+import org.opensearch.rest.RestHandler;
+import org.opensearch.script.ScriptService;
+import org.opensearch.security.sampleextension.actions.create.CreateSampleResourceAction;
+import org.opensearch.security.sampleextension.actions.create.CreateSampleResourceRestAction;
+import org.opensearch.security.sampleextension.actions.create.CreateSampleResourceTransportAction;
+import org.opensearch.security.sampleextension.actions.list.ListSampleResourceAction;
+import org.opensearch.security.sampleextension.actions.list.ListSampleResourceRestAction;
+import org.opensearch.security.sampleextension.actions.list.ListSampleResourceTransportAction;
+import org.opensearch.security.spi.ResourceSharingUtils;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.watcher.ResourceWatcherService;
+
+/**
+ * Sample Security Resource Sharing extension plugin.
+ *
+ * It use ".sample_extension_resources" index to manage its resources, and exposes a REST API
+ *
+ */
+public class SampleExtensionPlugin extends Plugin implements ActionPlugin, SystemIndexPlugin {
+    private static final Logger log = LogManager.getLogger(SampleExtensionPlugin.class);
+
+    public static final String RESOURCE_INDEX_NAME = ".sample_extension_resources";
+
+    private Client client;
+
+    @Override
+    public Collection<Object> createComponents(
+        Client client,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ResourceWatcherService resourceWatcherService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        Environment environment,
+        NodeEnvironment nodeEnvironment,
+        NamedWriteableRegistry namedWriteableRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<RepositoriesService> repositoriesServiceSupplier
+    ) {
+        this.client = client;
+        ResourceSharingUtils.getInstance().initialize(threadPool, client);
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(
+        Settings settings,
+        RestController restController,
+        ClusterSettings clusterSettings,
+        IndexScopedSettings indexScopedSettings,
+        SettingsFilter settingsFilter,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<DiscoveryNodes> nodesInCluster
+    ) {
+        return List.of(new CreateSampleResourceRestAction(), new ListSampleResourceRestAction());
+    }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return List.of(
+            new ActionHandler<>(CreateSampleResourceAction.INSTANCE, CreateSampleResourceTransportAction.class),
+            new ActionHandler<>(ListSampleResourceAction.INSTANCE, ListSampleResourceTransportAction.class)
+        );
+    }
+
+    @Override
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+        final SystemIndexDescriptor systemIndexDescriptor = new SystemIndexDescriptor(RESOURCE_INDEX_NAME, "Example index with resources");
+        return Collections.singletonList(systemIndexDescriptor);
+    }
+}

sample-extension-plugin/src/main/java/org/opensearch/security/sampleextension/actions/create/CreateSampleResourceAction.java

@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.security.sampleextension.actions.create;
+
+import org.opensearch.action.ActionType;
+import org.opensearch.security.spi.actions.CreateResourceResponse;
+
+/**
+ * Action to create a sample resource
+ */
+public class CreateSampleResourceAction extends ActionType<CreateResourceResponse> {
+    /**
+     * Create sample resource action instance
+     */
+    public static final CreateSampleResourceAction INSTANCE = new CreateSampleResourceAction();
+    /**
+     * Create sample resource action name
+     */
+    public static final String NAME = "cluster:admin/sampleresource/create";
+
+    private CreateSampleResourceAction() {
+        super(NAME, CreateResourceResponse::new);
+    }
+}

sample-extension-plugin/src/main/java/org/opensearch/security/sampleextension/actions/create/CreateSampleResourceRequest.java

@@ -0,0 +1,55 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.security.sampleextension.actions.create;
+
+import java.io.IOException;
+
+import org.opensearch.action.ActionRequest;
+import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.security.spi.Resource;
+
+/**
+ * Request object for CreateSampleResource transport action
+ */
+public class CreateSampleResourceRequest extends ActionRequest {
+
+    private final Resource resource;
+
+    /**
+     * Default constructor
+     */
+    public CreateSampleResourceRequest(Resource resource) {
+        this.resource = resource;
+    }
+
+    /**
+     * Constructor with stream input
+     * @param in the stream input
+     * @throws IOException IOException
+     */
+    public CreateSampleResourceRequest(final StreamInput in) throws IOException {
+        this.resource = new SampleResource(in);
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        resource.writeTo(out);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    public Resource getResource() {
+        return this.resource;
+    }
+}