From b6a34ad0f29f05d5a5a38b07b443263e4d264255 Mon Sep 17 00:00:00 2001 From: David Estes Date: Tue, 26 Apr 2016 10:03:55 -0400 Subject: [PATCH 01/16] fixed bug where test package was being packaged with plugin --- build.gradle | 13 ++++++++++++- gradle/publish.gradle | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 606318a3..4078b6be 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { id "com.jfrog.bintray" version "1.2" } -version "1.0.0.1" +version "1.0.0.2" group "org.grails.plugins" apply plugin: 'maven-publish' @@ -54,6 +54,17 @@ jar { exclude "test/**/**" } +sourceSets { + main { + java { + exclude 'test/**' + } + groovy { + exclude 'test/**' + } + } +} + dependencies { provided 'org.springframework.boot:spring-boot-starter-logging' provided "org.springframework.boot:spring-boot-starter-actuator" diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 62a29e01..b57cafdd 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -17,6 +17,16 @@ if (bintrayProperties.exists()) { desc = "An Elasticsearch plugin for Grails" developers = [noam: 'Noam Y. Tenne', macrcos: 'Marcos Carceles', puneet: 'Puneet Behl', james: 'James Kleeh'] } +} else { + grailsPublish { + githubSlug = 'noamt/elasticsearch-grails-plugin' + license { + name = 'Apache-2.0' + } + title = "Elasticserach" + desc = "An Elasticsearch plugin for Grails" + developers = [noam: 'Noam Y. Tenne', macrcos: 'Marcos Carceles', puneet: 'Puneet Behl', james: 'James Kleeh'] + } } // Publish gh-pages on github From 939770e124bc6b68fe67d7e5b9bce9ec95d19549 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Thu, 12 May 2016 16:26:34 +0200 Subject: [PATCH 02/16] Test with JDK 8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d237a596..c40d05d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: groovy jdk: -- oraclejdk7 +- oraclejdk8 before_script: - rm -rf target script: travis_wait 60 ./travis-build.sh From f83a4985d17a25848e6e7925f49471760c3dd7f9 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Fri, 13 May 2016 09:37:09 +0200 Subject: [PATCH 03/16] ES 2.3.2 upgrade WIP --- README.asciidoc | 2 +- build.gradle | 30 ++- gradle/wrapper/gradle-wrapper.properties | 4 +- grails-app/conf/application.yml | 2 +- grails-app/conf/plugin.groovy | 4 +- .../domain/test/upperCase/UpperCase.groovy | 2 +- .../ElasticSearchAdminService.groovy | 49 +---- .../elasticsearch/ElasticSearchService.groovy | 11 +- src/docs/introduction/history.adoc | 6 + src/docs/mapping/mappingMigrations.adoc | 9 +- .../DynamicMethodsIntegrationSpec.groovy | 8 +- ...ElasticSearchServiceIntegrationSpec.groovy | 24 +-- ...ainClassUnmarshallerIntegrationSpec.groovy | 8 +- .../mapping/MappingMigrationSpec.groovy | 181 ++++++++++++++---- .../ClientNodeFactoryBean.groovy | 55 +++++- .../ElasticSearchMappingFactory.groovy | 2 +- .../mapping/MappingMigrationManager.groovy | 44 +++-- .../mapping/MappingMigrationStrategy.groovy | 2 +- .../SearchableClassMappingConfigurator.groovy | 5 +- .../util/DomainDynamicMethodsUtils.groovy | 5 +- 20 files changed, 282 insertions(+), 171 deletions(-) diff --git a/README.asciidoc b/README.asciidoc index 571ba84f..c641c8da 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -16,7 +16,7 @@ Edit your project's +build.gradle+ file, by adding the plugin's dependency decla ---- dependencies { ... - compile "org.grails.plugins:elasticsearch:1.0.0.1" + compile "org.grails.plugins:elasticsearch:2.3.0" ... } diff --git a/build.gradle b/build.gradle index 4078b6be..cf83b866 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { id "com.jfrog.bintray" version "1.2" } -version "1.0.0.2" +version "2.3.0" group "org.grails.plugins" apply plugin: 'maven-publish' @@ -30,11 +30,11 @@ apply plugin: "org.grails.grails-gsp" ext { grailsVersion = project.grailsVersion gradleWrapperVersion = project.gradleWrapperVersion - elasticsearchVersion = "1.0.0.1" + elasticsearchVersion = "2.3.0" } -sourceCompatibility = 1.7 -targetCompatibility = 1.7 +sourceCompatibility = 1.8 +targetCompatibility = 1.8 repositories { mavenLocal() @@ -54,15 +54,14 @@ jar { exclude "test/**/**" } -sourceSets { - main { - java { - exclude 'test/**' - } - groovy { - exclude 'test/**' - } - } +groovydoc { + exclude "test/**" + exclude "test/**/**" +} + +sourcesJar { + exclude "test/**" + exclude "test/**/**" } dependencies { @@ -78,10 +77,7 @@ dependencies { compile "org.grails.plugins:cache" compile "org.hibernate:hibernate-ehcache" - testRuntime('org.elasticsearch:elasticsearch-groovy:1.6.0') { - exclude module: 'groovy-all' - } - compile "org.elasticsearch:elasticsearch-mapper-attachments:2.6.0" + compile 'org.elasticsearch:elasticsearch:2.3.2' testRuntime 'com.spatial4j:spatial4j:0.4.1' testCompile 'com.vividsolutions:jts:1.13' diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3869368d..9abdf608 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Fri Nov 27 23:09:32 CET 2015 +#Thu May 12 16:30:39 CEST 2016 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-2.9-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-2.9-all.zip diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index 129b40be..a3fa4cc6 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -92,7 +92,7 @@ environments: transport.sniff: true datastoreImpl: hibernateDatastore index: - store.type: memory + store.type: simplefs analysis: filter: replace_synonyms: diff --git a/grails-app/conf/plugin.groovy b/grails-app/conf/plugin.groovy index c8ae23cd..bea40ed0 100644 --- a/grails-app/conf/plugin.groovy +++ b/grails-app/conf/plugin.groovy @@ -102,8 +102,10 @@ environments { test { elasticSearch { client.mode = 'local' - index.store.type = 'memory' // store local node in memory and not on disk + index.store.type = 'simplefs' // store local node in memory and not on disk } + + path.plugins = 'src/test/resources/plugins' } production { elasticSearch.client.mode = 'node' diff --git a/grails-app/domain/test/upperCase/UpperCase.groovy b/grails-app/domain/test/upperCase/UpperCase.groovy index 26765b7f..a36051f9 100644 --- a/grails-app/domain/test/upperCase/UpperCase.groovy +++ b/grails-app/domain/test/upperCase/UpperCase.groovy @@ -1,7 +1,7 @@ package test.upperCase class UpperCase { - def name; + String name static mapping = { autoImport(false) diff --git a/grails-app/services/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy b/grails-app/services/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy index 8df6358f..9ecaf0f3 100644 --- a/grails-app/services/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy +++ b/grails-app/services/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy @@ -7,11 +7,11 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest -import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse +import org.elasticsearch.action.support.broadcast.BroadcastResponse import org.elasticsearch.client.Client import org.elasticsearch.client.Requests +import org.elasticsearch.cluster.health.ClusterHealthStatus import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -44,7 +44,7 @@ class ElasticSearchAdminService { // Refresh ES elasticSearchHelper.withElasticSearch { Client client -> - BroadcastOperationResponse response + BroadcastResponse response if (!indices) { response = client.admin().indices().refresh(Requests.refreshRequest()).actionGet() } else { @@ -135,21 +135,6 @@ class ElasticSearchAdminService { } } - /** - * Deletes a mapping on an index - * @param index The index the mapping will be deleted on - * @param type The type which mapping will be deleted - */ - void deleteMapping(String index, String type) { - log.info("Deleting Elasticsearch mapping for ${index} and type ${type} ...") - elasticSearchHelper.withElasticSearch { Client client -> - client.admin().indices().deleteMapping( - new DeleteMappingRequest(index). - types(type) - ).actionGet() - } - } - /** * Creates mappings on a type * @param index The index where the mapping is being created @@ -168,7 +153,7 @@ class ElasticSearchAdminService { } /** - * Check whether a mpping exists + * Check whether a mapping exists * @param index The name of the index to check on * @param type The type which mapping is being checked * @return true if the mapping exists @@ -254,7 +239,7 @@ class ElasticSearchAdminService { String indexPointedBy(String alias) { elasticSearchHelper.withElasticSearch { Client client -> def index = client.admin().indices().getAliases(new GetAliasesRequest().aliases([alias] as String[])).actionGet().getAliases()?.find { - it.value.element.alias() == alias + alias in it.value*.alias() }?.key if (!index) { LOG.debug("Alias ${alias} does not exist") @@ -363,32 +348,12 @@ class ElasticSearchAdminService { getLatestVersion(index) + 1 } - /** - * Waits for an index to be on Yellow status - * @param index - * @return - */ - def waitForIndex(index) { - elasticSearchHelper.withElasticSearch { Client client -> - try { - LOG.debug("Waiting at least yellow status on ${index}") - client.admin().cluster().prepareHealth(index) - .setWaitForYellowStatus() - .execute().actionGet() - } catch (Exception e) { - // ignore any exceptions due to non-existing index. - LOG.debug('Index health', e) - } - } - } - /** * Waits for the cluster to be on Yellow status */ - void waitForClusterYellowStatus() { + void waitForClusterStatus(ClusterHealthStatus status=ClusterHealthStatus.YELLOW) { elasticSearchHelper.withElasticSearch { Client client -> - ClusterHealthResponse response = client.admin().cluster().health( - new ClusterHealthRequest([] as String[]).waitForYellowStatus()).actionGet() + ClusterHealthResponse response = client.admin().cluster().health(new ClusterHealthRequest([] as String[]).waitForStatus(status)).actionGet() LOG.debug("Cluster status: ${response.status}") } } diff --git a/grails-app/services/grails/plugins/elasticsearch/ElasticSearchService.groovy b/grails-app/services/grails/plugins/elasticsearch/ElasticSearchService.groovy index 2513ff8d..5baad8ef 100755 --- a/grails-app/services/grails/plugins/elasticsearch/ElasticSearchService.groovy +++ b/grails-app/services/grails/plugins/elasticsearch/ElasticSearchService.groovy @@ -25,7 +25,6 @@ import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.action.search.SearchType import org.elasticsearch.action.support.QuerySourceBuilder import org.elasticsearch.client.Client -import org.elasticsearch.index.query.FilterBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryStringQueryBuilder import org.elasticsearch.search.SearchHit @@ -36,7 +35,7 @@ import org.elasticsearch.search.sort.SortOrder import org.slf4j.Logger import org.slf4j.LoggerFactory -import static org.elasticsearch.index.query.QueryBuilders.queryString +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery import static org.elasticsearch.index.query.QueryStringQueryBuilder.Operator class ElasticSearchService implements GrailsApplicationAware { @@ -337,7 +336,7 @@ class ElasticSearchService implements GrailsApplicationAware { request.source(new GXContentBuilder().buildAsBytes(query)) } else { Operator defaultOperator = params['default_operator'] ?: Operator.AND - QueryStringQueryBuilder builder = queryString(query).defaultOperator(defaultOperator) + QueryStringQueryBuilder builder = queryStringQuery(query).defaultOperator(defaultOperator) if (params.analyzer) { builder.analyzer(params.analyzer) } @@ -352,7 +351,7 @@ class ElasticSearchService implements GrailsApplicationAware { * * @param params The query parameters * @param query The search query, whether a String, a Closure or a QueryBuilder - * @param filter The search filter, whether a Closure or a FilterBuilder + * @param filter The search filter, whether a Closure or a QueryBuilder * @return The SearchRequest instance */ private SearchRequest buildSearchRequest(query, filter, Map params) { @@ -405,7 +404,7 @@ class ElasticSearchService implements GrailsApplicationAware { SearchSourceBuilder setQueryInSource(SearchSourceBuilder source, String query, Map params = [:]) { Operator defaultOperator = params['default_operator'] ?: Operator.AND - QueryStringQueryBuilder builder = queryString(query).defaultOperator(defaultOperator) + QueryStringQueryBuilder builder = queryStringQuery(query).defaultOperator(defaultOperator) if (params.analyzer) { builder.analyzer(params.analyzer) } @@ -424,7 +423,7 @@ class ElasticSearchService implements GrailsApplicationAware { source.postFilter(new GXContentBuilder().buildAsBytes(filter)) } - SearchSourceBuilder setFilterInSource(SearchSourceBuilder source, FilterBuilder filter, Map params = [:]){ + SearchSourceBuilder setFilterInSource(SearchSourceBuilder source, QueryBuilder filter, Map params = [:]){ source.postFilter(filter) } diff --git a/src/docs/introduction/history.adoc b/src/docs/introduction/history.adoc index da6fe314..929603e4 100644 --- a/src/docs/introduction/history.adoc +++ b/src/docs/introduction/history.adoc @@ -2,6 +2,12 @@ ==== Grails 3.x version +* May 13, 2016 +** 2.3.0 +*** Upgraded to Elasticsearch 2.3.2, versioning now follow pattern described on https://github.com/noamt/elasticsearch-grails-plugin/issues/141[Issue 141] +*** Removed test classes from plugin distribution +*** elasticsearch-groovy client removed (no available compatible version, pending on https://github.com/elastic/elasticsearch-groovy/issues/35[Issue 35]) + * April 18, 2016 ** 1.0.0.1 *** Add ability to change search method name in domain class via config diff --git a/src/docs/mapping/mappingMigrations.adoc b/src/docs/mapping/mappingMigrations.adoc index 08fb8d9d..ecc5581f 100644 --- a/src/docs/mapping/mappingMigrations.adoc +++ b/src/docs/mapping/mappingMigrations.adoc @@ -10,7 +10,8 @@ It is important to highlight that not all type mapping changes will result on a The migration strategy is defined by the `elasticSearch.migration.strategy` configuration property and it accepts three values: - `'none'` -- `'delete'` +- `'deleteIndex'` +- [line-through]#`'delete'` is no longer supported since the upgrade to Elasticsearch > 2.0# - `'alias'` The default strategy is `'alias'` as it is the only strategy that can achieve zero-downtime migrations and thus http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/[recommended by Elasticsearch] @@ -24,12 +25,12 @@ It will be responsibility for the application administrator to manually fix the This configuration was left as a backwards compatibility and it will prevent the application from booting successfully, therefore we *discourage teams from using this option*. -==== Migration Strategy 'delete' +==== Migration Strategy 'deleteIndex' -When choosing this option, when a conflict occurs installing mapping, the application will delete the existing mapping for the type, alongside with all content indexed on that index and type and recreated the mapping. There are a couple of important details on this information: +When choosing this option, when a conflict occurs installing mapping, the application will delete the existing index for the type, *alongside with all content indexed on that index (including content from other mappings)* and it will recreate the index and all its mappings. There are a couple of important details on this information: -- Only documents indexed on the conflicting mapping will be deleted, any other document on a different mapping on the same (or other) index will remain untouched. - Deleted documents can be automatically reindexed on startup by using the `elasticSearch.bulkIndexOnStartup` configuration property (See below) +- New indices will be created with a version number and the right aliases, to make them compatible with potential future 'alias' migrations (without requiring additional index deletions) - Using this configuration there will always be a time window (between deletion and reindexation) where documents can't be found by search, therefore this option cannot achieve a *zero-downtime* deployment See <> below for more details on automatic indexing. diff --git a/src/integration-test/groovy/grails/plugins/elasticsearch/DynamicMethodsIntegrationSpec.groovy b/src/integration-test/groovy/grails/plugins/elasticsearch/DynamicMethodsIntegrationSpec.groovy index 293e0ce1..e5d1d4be 100644 --- a/src/integration-test/groovy/grails/plugins/elasticsearch/DynamicMethodsIntegrationSpec.groovy +++ b/src/integration-test/groovy/grails/plugins/elasticsearch/DynamicMethodsIntegrationSpec.groovy @@ -2,8 +2,6 @@ package grails.plugins.elasticsearch import grails.test.mixin.integration.Integration import grails.transaction.Rollback -import org.elasticsearch.index.query.FilterBuilder -import org.elasticsearch.index.query.FilterBuilders import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.springframework.beans.factory.annotation.Autowired @@ -109,7 +107,7 @@ class DynamicMethodsIntegrationSpec extends Specification { when: QueryBuilder query = QueryBuilders.matchAllQuery() - FilterBuilder filter = FilterBuilders.termFilter("url", "http://www.nicenicejpg.com/100") + QueryBuilder filter = QueryBuilders.termQuery("url", "http://www.nicenicejpg.com/100") def results = Photo.search(query, filter) then: @@ -117,14 +115,14 @@ class DynamicMethodsIntegrationSpec extends Specification { results.searchResults[0].name == "Captain Kirk" } - void "can search and filter using Dynamic Methods and a FilterBuilder"() { + void "can search and filter using Dynamic Methods and a QueryBuilder"() { given: elasticSearchAdminService.refresh() expect: elasticSearchService.search('Captain', [indices: Photo, types: Photo]).total == 5 when: - FilterBuilder filter = FilterBuilders.termFilter("url", "http://www.nicenicejpg.com/100") + QueryBuilder filter = QueryBuilders.termQuery("url", "http://www.nicenicejpg.com/100") def results = Photo.search({ match(name: "Captain") }, filter) diff --git a/src/integration-test/groovy/grails/plugins/elasticsearch/ElasticSearchServiceIntegrationSpec.groovy b/src/integration-test/groovy/grails/plugins/elasticsearch/ElasticSearchServiceIntegrationSpec.groovy index f26d2863..ca123e16 100644 --- a/src/integration-test/groovy/grails/plugins/elasticsearch/ElasticSearchServiceIntegrationSpec.groovy +++ b/src/integration-test/groovy/grails/plugins/elasticsearch/ElasticSearchServiceIntegrationSpec.groovy @@ -16,8 +16,6 @@ import org.elasticsearch.cluster.ClusterState import org.elasticsearch.cluster.metadata.IndexMetaData import org.elasticsearch.cluster.metadata.MappingMetaData import org.elasticsearch.common.unit.DistanceUnit -import org.elasticsearch.index.query.FilterBuilder -import org.elasticsearch.index.query.FilterBuilders import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.aggregations.AggregationBuilders @@ -274,7 +272,7 @@ class ElasticSearchServiceIntegrationSpec extends Specification { when: 'a geo distance filter search is performed' Map params = [indices: Building, types: Building] - Closure query = null + Closure query = QueryBuilders.matchAllQuery() def location = '50, 13' Closure filter = { @@ -307,11 +305,7 @@ class ElasticSearchServiceIntegrationSpec extends Specification { elasticSearchAdminService.refresh() when: 'searching for a price' - def result = elasticSearchService.search(null as Closure, { - range { - "price"(gte: 1.99, lte: 2.3) - } - }) + def result = elasticSearchService.search(QueryBuilders.matchAllQuery(), QueryBuilders.rangeQuery("price").gte(1.99).lte(2.3)) then: "the result should be product 'wurm'" result.total == 1 @@ -321,8 +315,8 @@ class ElasticSearchServiceIntegrationSpec extends Specification { void 'searching with a FilterBuilder filter and a Closure query'() { when: 'searching for a price' - FilterBuilder filter = FilterBuilders.rangeFilter("price").gte(1.99).lte(2.3) - def result = elasticSearchService.search(null as Closure, filter) + QueryBuilder filter = QueryBuilders.rangeQuery("price").gte(1.99).lte(2.3) + def result = elasticSearchService.search(QueryBuilders.matchAllQuery(), filter) then: "the result should be product 'wurm'" result.total == 1 @@ -332,8 +326,8 @@ class ElasticSearchServiceIntegrationSpec extends Specification { void 'searching with a FilterBuilder filter and a QueryBuilder query'() { when: 'searching for a price' - FilterBuilder filter = FilterBuilders.rangeFilter("price").gte(1.99).lte(2.3) - def result = elasticSearchService.search(null as QueryBuilder, filter) + QueryBuilder filter = QueryBuilders.rangeQuery("price").gte(1.99).lte(2.3) + def result = elasticSearchService.search(QueryBuilders.matchAllQuery(), filter) then: "the result should be product 'wurm'" result.total == 1 @@ -427,7 +421,7 @@ class ElasticSearchServiceIntegrationSpec extends Specification { when: def result = elasticSearchService.search( QueryBuilders.hasParentQuery('store', QueryBuilders.matchQuery('owner', 'Horst')), - null as Closure, + QueryBuilders.matchAllQuery(), [indices: Department, types: Department] ) @@ -527,7 +521,7 @@ class ElasticSearchServiceIntegrationSpec extends Specification { when: 'a geo distance search is performed' Map params = [indices: Building, types: Building] - Closure query = null + QueryBuilder query = QueryBuilders.matchAllQuery() def location = [lat: 48.141, lon: 11.57] Closure filter = { @@ -587,7 +581,7 @@ class ElasticSearchServiceIntegrationSpec extends Specification { order(SortOrder.ASC) Map params = [indices: Building, types: Building, sort: sortBuilder] - Closure query = null + QueryBuilder query = QueryBuilders.matchAllQuery() def location = [lat: 48.141, lon: 11.57] Closure filter = { diff --git a/src/integration-test/groovy/grails/plugins/elasticsearch/conversion/unmarshall/DomainClassUnmarshallerIntegrationSpec.groovy b/src/integration-test/groovy/grails/plugins/elasticsearch/conversion/unmarshall/DomainClassUnmarshallerIntegrationSpec.groovy index fdf6752e..78f30081 100644 --- a/src/integration-test/groovy/grails/plugins/elasticsearch/conversion/unmarshall/DomainClassUnmarshallerIntegrationSpec.groovy +++ b/src/integration-test/groovy/grails/plugins/elasticsearch/conversion/unmarshall/DomainClassUnmarshallerIntegrationSpec.groovy @@ -3,7 +3,7 @@ package grails.plugins.elasticsearch.conversion.unmarshall import grails.test.mixin.integration.Integration import grails.core.GrailsApplication import org.elasticsearch.common.bytes.BytesArray -import org.elasticsearch.common.text.StringAndBytesText +import org.elasticsearch.common.text.Text import org.elasticsearch.search.internal.InternalSearchHit import org.elasticsearch.search.internal.InternalSearchHits import grails.plugins.elasticsearch.ElasticSearchContextHolder @@ -32,7 +32,7 @@ class DomainClassUnmarshallerIntegrationSpec extends Specification { def unmarshaller = new DomainClassUnmarshaller(elasticSearchContextHolder: elasticSearchContextHolder, grailsApplication: grailsApplication) given: 'a search hit with a geo_point' - InternalSearchHit hit = new InternalSearchHit(1, '1', new StringAndBytesText('building'), [:]) + InternalSearchHit hit = new InternalSearchHit(1, '1', new Text('building'), [:]) .sourceRef(new BytesArray('{"location":{"class":"test.GeoPoint","id":"2", "lat":53.0,"lon":10.0},"name":"WatchTower"}')) InternalSearchHit[] hits = [hit] def maxScore = 0.1534264087677002f @@ -55,7 +55,7 @@ class DomainClassUnmarshallerIntegrationSpec extends Specification { def unmarshaller = new DomainClassUnmarshaller(elasticSearchContextHolder: elasticSearchContextHolder, grailsApplication: grailsApplication) given: 'a search hit with a color with unhandled properties r-g-b' - InternalSearchHit hit = new InternalSearchHit(1, '1', new StringAndBytesText('color'), [:]) + InternalSearchHit hit = new InternalSearchHit(1, '1', new Text('color'), [:]) .sourceRef(new BytesArray('{"name":"Orange", "red":255, "green":153, "blue":0}')) InternalSearchHit[] hits = [hit] def maxScore = 0.1534264087677002f @@ -85,7 +85,7 @@ class DomainClassUnmarshallerIntegrationSpec extends Specification { def unmarshaller = new DomainClassUnmarshaller(elasticSearchContextHolder: elasticSearchContextHolder, grailsApplication: grailsApplication) given: 'a search hit with a circle, within it a color with an unhandled properties "red"' - InternalSearchHit hit = new InternalSearchHit(1, '1', new StringAndBytesText('circle'), [:]) + InternalSearchHit hit = new InternalSearchHit(1, '1', new Text('circle'), [:]) .sourceRef(new BytesArray('{"radius":7, "color":{"class":"test.Color", "id":"2", "name":"Orange", "red":255}}')) InternalSearchHit[] hits = [hit] def maxScore = 0.1534264087677002f diff --git a/src/integration-test/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationSpec.groovy b/src/integration-test/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationSpec.groovy index d02417b9..7e54cac0 100644 --- a/src/integration-test/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationSpec.groovy +++ b/src/integration-test/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationSpec.groovy @@ -95,12 +95,52 @@ class MappingMigrationSpec extends Specification { void "when there's a conflict and no strategy is selected an exception is thrown"() { given: "A Conflicting Catalog mapping (with nested as opposed to inner pages)" - createConflictingCatalogMapping() + //Delete existing Mapping + es.deleteIndex catalogMapping.indexName + //Create conflicting Mapping + catalogPagesMapping.addAttributes([component:true]) + searchableClassMappingConfigurator.installMappings([catalogMapping]) + //Restore initial state for next use + catalogPagesMapping.addAttributes([component:'inner']) + + expect: + es.indexExists(catalogMapping.indexName) + !es.aliasExists(catalogMapping.indexName) - and: "No Migration Configuration" + when: "No Migration Configuration" grailsApplication.config.elasticSearch.migration = [strategy: "none"] - when: + and: + searchableClassMappingConfigurator.installMappings([catalogMapping]) + + then: + thrown MappingException + } + + /* + * STRATEGY : delete + * Depreceated, throws Exception now + */ + + void "when there is a conflict and strategy is 'delete' an exception is thrown"() { + + given: "A Conflicting Catalog mapping (with nested as opposed to inner pages)" + //Delete existing Mapping + es.deleteIndex catalogMapping.indexName + //Create conflicting Mapping + catalogPagesMapping.addAttributes([component:true]) + searchableClassMappingConfigurator.installMappings([catalogMapping]) + //Restore initial state for next use + catalogPagesMapping.addAttributes([component:'inner']) + + expect: + es.indexExists(catalogMapping.indexName) + !es.aliasExists(catalogMapping.indexName) + + when: "Delete Configuration" + grailsApplication.config.elasticSearch.migration = [strategy: "delete"] + + and: searchableClassMappingConfigurator.installMappings([catalogMapping]) then: @@ -113,46 +153,65 @@ class MappingMigrationSpec extends Specification { * case 2: Incompatible Alias exists */ - void "when there's a conflict and strategy is 'delete' content is deleted"() { + void "when there is a conflict and strategy is 'deleteIndex' content is deleted"() { given: "A Conflicting Catalog mapping (with nested as opposed to inner pages)" - createConflictingCatalogMapping() + //Delete existing Mapping + es.deleteIndex catalogMapping.indexName + //Create conflicting Mapping + catalogPagesMapping.addAttributes([component:true]) + searchableClassMappingConfigurator.installMappings([catalogMapping]) + //Restore initial state for next use + catalogPagesMapping.addAttributes([component:'inner']) and: "Delete Configuration" - grailsApplication.config.elasticSearch.migration = [strategy: "delete"] + grailsApplication.config.elasticSearch.migration = [strategy: "deleteIndex"] and: "Existing content" new Catalog(company:"ACME", issue: 1).save(flush:true,failOnError: true) new Catalog(company:"ACME", issue: 2).save(flush:true,failOnError: true) + new Item(name:"Super Jump Spring Actioned Boots").save(flush:true,failOnError: true) elasticSearchService.index() elasticSearchAdminService.refresh() expect: + es.indexExists(catalogMapping.indexName) + !es.aliasExists(catalogMapping.indexName) + + and: es.indexPointedBy(catalogMapping.queryingIndex) == catalogMapping.indexName Catalog.count() == 2 Catalog.search("ACME").total == 2 + Item.count() == 1 + Item.search("Spring").total == 1 when: "Installing the conflicting mapping" searchableClassMappingConfigurator.installMappings([catalogMapping]) - then: "It succeeds" + then: "It succeeds -> The index is recreated" es.indexExists(catalogMapping.indexName) - es.indexPointedBy(catalogMapping.indexingIndex) == catalogMapping.indexName - es.indexPointedBy(catalogMapping.queryingIndex) == catalogMapping.indexName + + and: "It is a versioned alias to an 'alias' strategy compatible index" + es.aliasExists(catalogMapping.indexName) + es.indexPointedBy(catalogMapping.indexingIndex) == es.indexPointedBy(catalogMapping.indexName) + es.indexPointedBy(catalogMapping.queryingIndex) == es.indexPointedBy(catalogMapping.indexName) + es.versionIndex(catalogMapping.indexName, 0) == es.indexPointedBy(catalogMapping.indexName) es.mappingExists catalogMapping.indexName, catalogMapping.elasticTypeName - and: "Documents are lost on ES as mapping was recreated" + and: "Documents are lost on ES" Catalog.count() == 2 Catalog.search("ACME").total == 0 - and: "No alias was created" - !es.aliasExists(catalogMapping.indexName) + and: "Other documents on the same index are lost as well" + Item.count() == 1 + Item.search("Spring").total == 0 cleanup: Catalog.findAll().each { it.delete() } + Item.findAll().each { it.delete() } } - void "delete works on alias as well"() { + void "delete on alias throws Exception because delete is deprecated"() { given: "An alias pointing to a versioned index" es.deleteIndex catalogMapping.indexName @@ -160,17 +219,49 @@ class MappingMigrationSpec extends Specification { es.pointAliasTo catalogMapping.indexName, catalogMapping.indexName, 0 es.pointAliasTo catalogMapping.queryingIndex, catalogMapping.indexName es.pointAliasTo catalogMapping.indexingIndex, catalogMapping.indexName - searchableClassMappingConfigurator.configureAndInstallMappings() and: "A Conflicting Catalog mapping (with nested as opposed to inner pages)" - createConflictingCatalogMapping() + catalogPagesMapping.addAttributes([component:true]) + searchableClassMappingConfigurator.installMappings([catalogMapping, itemMapping]) + catalogPagesMapping.addAttributes([component:'inner']) and: "Delete Configuration" grailsApplication.config.elasticSearch.migration = [strategy: "delete"] + expect: + es.indexExists(catalogMapping.indexName, 0) + es.indexPointedBy(catalogMapping.indexName) == es.versionIndex(catalogMapping.indexName, 0) + es.indexPointedBy(catalogMapping.queryingIndex) == es.versionIndex(catalogMapping.indexName, 0) + es.indexPointedBy(catalogMapping.indexingIndex) == es.versionIndex(catalogMapping.indexName, 0) + + when: "Installing the conflicting mapping" + searchableClassMappingConfigurator.installMappings([catalogMapping]) + + then: "it fails" + thrown MappingException + } + + void "deleteIndex works on alias as well"() { + + given: "An alias pointing to a versioned index" + es.deleteIndex catalogMapping.indexName + es.createIndex catalogMapping.indexName, 0 + es.pointAliasTo catalogMapping.indexName, catalogMapping.indexName, 0 + es.pointAliasTo catalogMapping.queryingIndex, catalogMapping.indexName + es.pointAliasTo catalogMapping.indexingIndex, catalogMapping.indexName + + and: "A Conflicting Catalog mapping (with nested as opposed to inner pages)" + catalogPagesMapping.addAttributes([component:true]) + searchableClassMappingConfigurator.installMappings([catalogMapping, itemMapping]) + catalogPagesMapping.addAttributes([component:'inner']) + + and: "Delete Configuration" + grailsApplication.config.elasticSearch.migration = [strategy: "deleteIndex"] + and: "Existing content" new Catalog(company:"ACME", issue: 1).save(flush:true,failOnError: true) new Catalog(company:"ACME", issue: 2).save(flush:true,failOnError: true) + new Item(name:"Super Jump Spring Actioned Boots").save(flush:true,failOnError: true) elasticSearchService.index() elasticSearchAdminService.refresh() @@ -181,6 +272,8 @@ class MappingMigrationSpec extends Specification { es.indexPointedBy(catalogMapping.indexingIndex) == es.versionIndex(catalogMapping.indexName, 0) Catalog.count() == 2 Catalog.search("ACME").total == 2 + Item.count() == 1 + Item.search("Spring").total == 1 when: "Installing the conflicting mapping" searchableClassMappingConfigurator.installMappings([catalogMapping]) @@ -198,8 +291,13 @@ class MappingMigrationSpec extends Specification { Catalog.count() == 2 Catalog.search("ACME").total == 0 + and: "Other documents on the same index are lost as well" + Item.count() == 1 + Item.search("Spring").total == 0 + cleanup: Catalog.findAll().each { it.delete() } + Item.findAll().each { it.delete() } } /* @@ -239,13 +337,17 @@ class MappingMigrationSpec extends Specification { es.pointAliasTo catalogMapping.indexName, catalogMapping.indexName, 10 es.pointAliasTo catalogMapping.queryingIndex, catalogMapping.indexName es.pointAliasTo catalogMapping.indexingIndex, catalogMapping.indexName - searchableClassMappingConfigurator.configureAndInstallMappings() and: "Two different mapping conflicts on the same index" assert catalogMapping != itemMapping assert catalogMapping.indexName == itemMapping.indexName - createConflictingCatalogMapping() - createConflictingProductMapping() + //Create conflicting Mapping + catalogPagesMapping.addAttributes([component:true]) + itemSupplierMapping.addAttributes([component:true]) + searchableClassMappingConfigurator.installMappings([catalogMapping, itemMapping]) + //Restore initial state for next use + catalogPagesMapping.addAttributes([component:'inner']) + itemSupplierMapping.addAttributes([component:'inner']) and: "Alias Configuration" grailsApplication.config.elasticSearch.migration = [strategy: "alias"] @@ -280,8 +382,15 @@ class MappingMigrationSpec extends Specification { given: "Two different mapping conflicts on the same index" assert catalogMapping != itemMapping assert catalogMapping.indexName == itemMapping.indexName - createConflictingCatalogMapping() - createConflictingProductMapping() + //Delete previous index + es.deleteIndex(catalogMapping.indexName) + //Create conflicting Mapping + catalogPagesMapping.addAttributes([component:true]) + itemSupplierMapping.addAttributes([component:true]) + searchableClassMappingConfigurator.installMappings([catalogMapping, itemMapping]) + //Restore initial state for next use + catalogPagesMapping.addAttributes([component:'inner']) + itemSupplierMapping.addAttributes([component:'inner']) and: "Existing content" new Catalog(company:"ACME", issue: 1).save(flush:true,failOnError: true) @@ -354,10 +463,12 @@ class MappingMigrationSpec extends Specification { es.pointAliasTo catalogMapping.indexName, catalogMapping.indexName, 0 es.pointAliasTo catalogMapping.queryingIndex, catalogMapping.indexName es.pointAliasTo catalogMapping.indexingIndex, catalogMapping.indexName - searchableClassMappingConfigurator.configureAndInstallMappings() and: "A mapping conflict" - createConflictingCatalogMapping() + catalogPagesMapping.addAttributes([component:true]) + searchableClassMappingConfigurator.installMappings([catalogMapping, itemMapping]) + //Restore initial state for next use + catalogPagesMapping.addAttributes([component:'inner']) and: "Existing content" new Catalog(company:"ACME", issue: 1).save(flush:true,failOnError: true) @@ -367,6 +478,12 @@ class MappingMigrationSpec extends Specification { elasticSearchAdminService.refresh() expect: + es.indexExists(catalogMapping.indexName, 0) + es.indexPointedBy(catalogMapping.indexName) == es.versionIndex(catalogMapping.indexName, 0) + es.indexPointedBy(catalogMapping.queryingIndex) == es.versionIndex(catalogMapping.indexName, 0) + es.indexPointedBy(catalogMapping.indexingIndex) == es.versionIndex(catalogMapping.indexName, 0) + + and: Catalog.count() == 2 Catalog.search("ACME").total == 2 Item.count() == 1 @@ -431,24 +548,4 @@ class MappingMigrationSpec extends Specification { it.propertyName == "supplier" } } - - private void createConflictingCatalogMapping() { - //Delete existing Mapping - es.deleteMapping catalogMapping.indexName, catalogMapping.elasticTypeName - //Create conflicting Mapping - catalogPagesMapping.addAttributes([component:true]) - searchableClassMappingConfigurator.installMappings([catalogMapping]) - //Restore initial state for next use - catalogPagesMapping.addAttributes([component:'inner']) - } - - private void createConflictingProductMapping() { - //Delete existing Mapping - es.deleteMapping itemMapping.indexName, itemMapping.elasticTypeName - //Create conflicting Mapping - itemSupplierMapping.addAttributes([component:true]) - searchableClassMappingConfigurator.installMappings([itemMapping]) - //Restore initial state for next use - itemSupplierMapping.addAttributes([component:'inner']) - } } diff --git a/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy b/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy index b1d585b7..36c91a4a 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy @@ -17,13 +17,18 @@ package grails.plugins.elasticsearch import org.elasticsearch.client.transport.TransportClient -import org.elasticsearch.common.settings.ImmutableSettings +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.transport.InetSocketTransportAddress import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.FactoryBean +import org.springframework.core.io.Resource import org.springframework.core.io.support.PathMatchingResourcePatternResolver +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + import static org.elasticsearch.node.NodeBuilder.nodeBuilder class ClientNodeFactoryBean implements FactoryBean { @@ -46,8 +51,8 @@ class ClientNodeFactoryBean implements FactoryBean { def configFile = elasticSearchContextHolder.config.bootstrap.config.file if (configFile) { LOG.info "Looking for bootstrap configuration file at: $configFile" - def resource = new PathMatchingResourcePatternResolver().getResource(configFile) - nb.settings(ImmutableSettings.settingsBuilder().loadFromUrl(resource.URL)) + Resource resource = new PathMatchingResourcePatternResolver().getResource(configFile) + nb.settings(Settings.settingsBuilder().loadFromStream(configFile, resource.inputStream)) } def transportClient @@ -66,28 +71,48 @@ class ClientNodeFactoryBean implements FactoryBean { // Configure the client based on the client mode switch (clientMode) { case 'transport': - def transportSettings = ImmutableSettings.settingsBuilder() + def transportSettings = Settings.settingsBuilder() def transportSettingsFile = elasticSearchContextHolder.config.bootstrap.transportSettings.file if (transportSettingsFile) { - def resource = new PathMatchingResourcePatternResolver().getResource(transportSettingsFile) - transportSettings.loadFromUrl(resource.URL) + Resource resource = new PathMatchingResourcePatternResolver().getResource(transportSettingsFile) + transportSettings.loadFromStream(transportSettingsFile, resource.inputStream) } // Use the "sniff" feature of transport client ? if (elasticSearchContextHolder.config.client.transport.sniff) { - transportSettings.put("client.transport.sniff", true) + transportSettings.put("client.transport.sniff", false) } if (elasticSearchContextHolder.config.cluster.name) { transportSettings.put('cluster.name', elasticSearchContextHolder.config.cluster.name.toString()) } transportClient = new TransportClient(transportSettings) + boolean ip4Enabled = elasticSearchContextHolder.config.shield.ip4Enabled ?: true + boolean ip6Enabled = elasticSearchContextHolder.config.shield.ip6Enabled ?: false + + try { + def shield = Class.forName("org.elasticsearch.shield.ShieldPlugin") + transportClient = TransportClient.builder().addPlugin(shield).settings(transportSettings).build(); + LOG.info("Shield Enabled") + } catch (ClassNotFoundException e) { + transportClient = TransportClient.builder().settings(transportSettings).build() + } + // Configure transport addresses if (!elasticSearchContextHolder.config.client.hosts) { - transportClient.addTransportAddress(new InetSocketTransportAddress('localhost', 9300)) + transportClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress('localhost', 9300))) } else { elasticSearchContextHolder.config.client.hosts.each { - transportClient.addTransportAddress(new InetSocketTransportAddress(it.host, it.port)) + try { + for (InetAddress address : InetAddress.getAllByName(it.host)) { + if ((ip6Enabled && address instanceof Inet6Address) || (ip4Enabled && address instanceof Inet4Address)) { + LOG.info("Adding host: ${address}") + transportClient.addTransportAddress(new InetSocketTransportAddress(address, it.port)); + } + } + } catch (UnknownHostException e) { + LOG.error("Unable to get the host", e.getMessage()); + } } } break @@ -126,6 +151,10 @@ class ClientNodeFactoryBean implements FactoryBean { nb.settings().put('path.conf', confDirectory as String) } + def tmpDirectory = tmpDirectory() + LOG.info "Setting embedded ElasticSearch tmp dir to ${tmpDirectory}" + nb.settings().put("path.home", tmpDirectory) + nb.local(true) break @@ -195,4 +224,12 @@ class ClientNodeFactoryBean implements FactoryBean { node.close() // close() seems to be more appropriate than stop() } } + + private String tmpDirectory() { + String baseDirectory = System.getProperty("java.io.tmpdir") ?: '/tmp' + Path path = Files.createTempDirectory(Paths.get(baseDirectory), 'elastic-data-' + new Date().time) + File file = path.toFile() + file.deleteOnExit() + return file.absolutePath + } } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/ElasticSearchMappingFactory.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/ElasticSearchMappingFactory.groovy index 3d27d414..f2bd8b91 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/ElasticSearchMappingFactory.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/ElasticSearchMappingFactory.groovy @@ -105,7 +105,7 @@ class ElasticSearchMappingFactory { } propOptions.type = propType // See http://www.elasticsearch.com/docs/elasticsearch/mapping/all_field/ - if ((propType != 'object') && scm.isAll()) { + if (!(propType in ['object', 'attachment']) && scm.isAll()) { // does it make sense to include objects into _all? propOptions.include_in_all = !scpm.shouldExcludeFromAll() } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy index 4d7b9097..2c61e73f 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy @@ -7,9 +7,7 @@ import grails.plugins.elasticsearch.exception.MappingException import org.slf4j.Logger import org.slf4j.LoggerFactory -import static grails.plugins.elasticsearch.mapping.MappingMigrationStrategy.alias -import static grails.plugins.elasticsearch.mapping.MappingMigrationStrategy.delete -import static grails.plugins.elasticsearch.mapping.MappingMigrationStrategy.none +import static grails.plugins.elasticsearch.mapping.MappingMigrationStrategy.* /** * Created by @marcos-carceles on 26/01/15. @@ -28,10 +26,13 @@ class MappingMigrationManager { } def applyMigrations(MappingMigrationStrategy migrationStrategy, Map elasticMappings, List mappingConflicts, Map indexSettings) { - switch(migrationStrategy) { + switch (migrationStrategy) { case delete: - applyDeleteStrategy(elasticMappings, mappingConflicts) - break; + LOG.error("Delete a Mapping is no longer supported since Elasticsearch 2.0 (see https://www.elastic.co/guide/en/elasticsearch/reference/2.0/indices-delete-mapping.html)." + + " To prevent data loss, this strategy has been replaced by 'deleteIndex'") + throw new MappingException() + case deleteIndex: + applyDeleteIndexStrategy(elasticMappings, mappingConflicts, indexSettings) case alias: applyAliasStrategy(elasticMappings, mappingConflicts, indexSettings) break; @@ -41,13 +42,26 @@ class MappingMigrationManager { } } - def applyDeleteStrategy(Map elasticMappings, List mappingConflicts) { + def applyDeleteIndexStrategy(Map elasticMappings, List mappingConflicts, Map indexSettings) { + List deletedIndices = [] mappingConflicts.each { SearchableClassMapping scm = it.scm - es.deleteMapping scm.indexName, scm.elasticTypeName - es.createMapping scm.indexName, scm.elasticTypeName, elasticMappings[it.scm] - elasticSearchContextHolder.deletedOnMigration << scm.domainClass.clazz + if(!deletedIndices.contains(scm.indexName)) { + deletedIndices << scm.indexName + es.deleteIndex scm.indexName + int nextVersion = es.getNextVersion(scm.indexName) + es.createIndex scm.indexName, nextVersion, indexSettings + es.waitForIndex scm.indexName, nextVersion //Ensure it exists so later on mappings are created on the right version + es.pointAliasTo scm.indexName, scm.indexName, nextVersion + es.pointAliasTo scm.indexingIndex, scm.indexName, nextVersion + if(!esConfig.bulkIndexOnStartup) { //Otherwise, it will be done post content creation + if (!esConfig.migration.disableAliasChange) { + es.pointAliasTo scm.queryingIndex, scm.indexName, nextVersion + } + } + } } + rebuildMappings(elasticMappings, deletedIndices) } def applyAliasStrategy(Map elasticMappings, List mappingConflicts, Map indexSettings) { @@ -66,17 +80,18 @@ class MappingMigrationManager { boolean conflictOnAlias = es.aliasExists(scm.indexName) println "Conflict: $conflictOnAlias" println "Config: $esConfig.migration.aliasReplacesIndex" - if(conflictOnAlias || esConfig.migration.aliasReplacesIndex ) { + if (conflictOnAlias || esConfig.migration.aliasReplacesIndex) { int nextVersion = es.getNextVersion(scm.indexName) if (!conflictOnAlias) { es.deleteIndex(scm.indexName) } es.createIndex scm.indexName, nextVersion, indexSettings - es.waitForIndex scm.indexName, nextVersion //Ensure it exists so later on mappings are created on the right version + es.waitForIndex scm.indexName, nextVersion + //Ensure it exists so later on mappings are created on the right version es.pointAliasTo scm.indexName, scm.indexName, nextVersion es.pointAliasTo scm.indexingIndex, scm.indexName, nextVersion - if(!esConfig.bulkIndexOnStartup) { //Otherwise, it will be done post content creation + if (!esConfig.bulkIndexOnStartup) { //Otherwise, it will be done post content creation if (!conflictOnAlias || !esConfig.migration.disableAliasChange) { es.pointAliasTo scm.queryingIndex, scm.indexName, nextVersion } @@ -93,7 +108,8 @@ class MappingMigrationManager { //Recreate the mappings for all the indexes that were changed elasticMappings.each { SearchableClassMapping scm, elasticMapping -> if (migratedIndices.contains(scm.indexName)) { - elasticSearchContextHolder.deletedOnMigration << scm.domainClass.clazz //Mark it for potential content index on Bootstrap + elasticSearchContextHolder.deletedOnMigration << scm.domainClass.clazz + //Mark it for potential content index on Bootstrap if (scm.isRoot()) { int newVersion = es.getLatestVersion(scm.indexName) String indexName = es.versionIndex(scm.indexName, newVersion) diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationStrategy.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationStrategy.groovy index 53632e84..aea63939 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationStrategy.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationStrategy.groovy @@ -4,5 +4,5 @@ package grails.plugins.elasticsearch.mapping * Created by @marcos-carceles on 22/12/14. */ enum MappingMigrationStrategy { - none, delete, alias + none, delete, deleteIndex, alias } \ No newline at end of file diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy index b1d68143..3c9f8d22 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy @@ -16,6 +16,7 @@ package grails.plugins.elasticsearch.mapping +import org.elasticsearch.cluster.health.ClusterHealthStatus import org.grails.core.artefact.DomainClassArtefactHandler import grails.core.GrailsApplication import grails.core.GrailsClass @@ -133,7 +134,7 @@ class SearchableClassMappingConfigurator { mmm.applyMigrations(migrationStrategy, elasticMappings, mappingConflicts, indexSettings) } - es.waitForClusterYellowStatus() + es.waitForClusterStatus(ClusterHealthStatus.YELLOW) } @@ -145,7 +146,7 @@ class SearchableClassMappingConfigurator { */ private boolean createIndexWithReadAndWrite(MappingMigrationStrategy strategy, SearchableClassMapping scm, Map indexSettings) throws RemoteTransportException { // Could be blocked on index level, thus wait. - es.waitForIndex(scm.indexName) + es.waitForClusterStatus(ClusterHealthStatus.YELLOW) if(!es.indexExists(scm.indexName)) { LOG.debug("Index ${scm.indexName} does not exists, initiating creation...") if (strategy == alias) { diff --git a/src/main/groovy/grails/plugins/elasticsearch/util/DomainDynamicMethodsUtils.groovy b/src/main/groovy/grails/plugins/elasticsearch/util/DomainDynamicMethodsUtils.groovy index c7d93882..9f5e50fb 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/util/DomainDynamicMethodsUtils.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/util/DomainDynamicMethodsUtils.groovy @@ -21,7 +21,6 @@ import grails.plugins.elasticsearch.ElasticSearchService import grails.plugins.elasticsearch.mapping.SearchableClassMapping import org.apache.commons.logging.LogFactory import grails.plugins.elasticsearch.exception.IndexException -import org.elasticsearch.index.query.FilterBuilder import org.elasticsearch.index.query.QueryBuilder class DomainDynamicMethodsUtils { @@ -91,10 +90,10 @@ class DomainDynamicMethodsUtils { domain.metaClass.'static'."$searchMethodName" << { QueryBuilder q, f = null, Map params = [:] -> elasticSearchService.search(q, f, params + indexAndType) } - domain.metaClass.'static'."$searchMethodName" << { Map params, QueryBuilder q, FilterBuilder f -> + domain.metaClass.'static'."$searchMethodName" << { Map params, QueryBuilder q, QueryBuilder f -> elasticSearchService.search(params + indexAndType, q, f) } - domain.metaClass.'static'."$searchMethodName" << { QueryBuilder q, FilterBuilder f, Map params = [:] -> + domain.metaClass.'static'."$searchMethodName" << { QueryBuilder q, QueryBuilder f, Map params = [:] -> elasticSearchService.search(q, f, params + indexAndType) } From 1777e2da959ce284620d9b7b17e30834375d880e Mon Sep 17 00:00:00 2001 From: Marcos Carceles Date: Mon, 16 May 2016 13:37:26 +0200 Subject: [PATCH 04/16] Update README.asciidoc --- README.asciidoc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.asciidoc b/README.asciidoc index c641c8da..8e3dc141 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -22,6 +22,29 @@ dependencies { } ---- + +=== Versioning + +|=== +|Plugin Version | Grails Version | Elasticsearch Version + +|2.3.x +|3.y +|2.3.z + +|1.0.0.x +|3.y +|1.6.z + +|0.1.0.x +|2.y +|2.1.z + +|0.0.4.x +|2.y +|1.6.z +|=== + === Ohs Nos! I'm hitting a +java.lang.Exception: No datastore implementation specified+ This error means that you probably haven't specified your datastore type in your +application.groovy+ or +application.yml+ file. From 17c8b98c1e56e1248708d79452ecba68b7966a41 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Wed, 1 Jun 2016 16:24:02 +0200 Subject: [PATCH 05/16] Upgraded to ES 2.3.2: * Explicitly setting plugins on local node * Handling Elasticsearch 2.0 breaking changes (_parent type) * Elasticsearch mappings created alongside index on atomic operation --- build.gradle | 6 +- gradle.properties | 2 +- grails-app/conf/application.yml | 1 - grails-app/conf/plugin.groovy | 2 - .../ElasticSearchAdminService.groovy | 9 +- ...ElasticSearchServiceIntegrationSpec.groovy | 11 +- .../ClientNodeFactoryBean.groovy | 47 +++++--- .../mapping/MappingMigrationManager.groovy | 104 +++++++----------- .../mapping/SearchableClassMapping.groovy | 8 +- .../SearchableClassMappingConfigurator.groovy | 80 ++++++++------ 10 files changed, 140 insertions(+), 130 deletions(-) diff --git a/build.gradle b/build.gradle index cf83b866..d1ad397f 100644 --- a/build.gradle +++ b/build.gradle @@ -64,6 +64,8 @@ sourcesJar { exclude "test/**/**" } +def elasticsearchVersion = '2.3.3' + dependencies { provided 'org.springframework.boot:spring-boot-starter-logging' provided "org.springframework.boot:spring-boot-starter-actuator" @@ -77,7 +79,7 @@ dependencies { compile "org.grails.plugins:cache" compile "org.hibernate:hibernate-ehcache" - compile 'org.elasticsearch:elasticsearch:2.3.2' + compile "org.elasticsearch:elasticsearch:${elasticsearchVersion}" testRuntime 'com.spatial4j:spatial4j:0.4.1' testCompile 'com.vividsolutions:jts:1.13' @@ -85,6 +87,8 @@ dependencies { testCompile "org.grails:grails-plugin-testing" console "org.grails:grails-console" + + compile "org.elasticsearch.plugin:mapper-attachments:${elasticsearchVersion}" } task wrapper(type: Wrapper) { diff --git a/gradle.properties b/gradle.properties index d388b8bc..69156480 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ -grailsVersion=3.1.1 +grailsVersion=3.1.8 gradleWrapperVersion=2.9 diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index a3fa4cc6..ea9053de 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -105,7 +105,6 @@ environments: repl_analyzer: tokenizer: standard filter: ['lowercase', 'replace_synonyms'] - production: elasticSearch: client: diff --git a/grails-app/conf/plugin.groovy b/grails-app/conf/plugin.groovy index bea40ed0..71da4542 100644 --- a/grails-app/conf/plugin.groovy +++ b/grails-app/conf/plugin.groovy @@ -104,8 +104,6 @@ environments { client.mode = 'local' index.store.type = 'simplefs' // store local node in memory and not on disk } - - path.plugins = 'src/test/resources/plugins' } production { elasticSearch.client.mode = 'node' diff --git a/grails-app/services/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy b/grails-app/services/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy index 9ecaf0f3..846faed5 100644 --- a/grails-app/services/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy +++ b/grails-app/services/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy @@ -182,7 +182,7 @@ class ElasticSearchAdminService { * @param index The name of the index * @param settings The index settings (ie. number of shards) */ - void createIndex(String index, Map settings=null) { + void createIndex(String index, Map settings=null, Map esMappings = [:]) { LOG.debug "Creating index ${index} ..." elasticSearchHelper.withElasticSearch { Client client -> @@ -190,6 +190,9 @@ class ElasticSearchAdminService { if (settings) { builder.setSettings(settings) } + esMappings.each { String type, Map elasticMapping -> + builder.addMapping(type, elasticMapping) + } builder.execute().actionGet() } } @@ -200,9 +203,9 @@ class ElasticSearchAdminService { * @param version the version number, if provided _v will be used * @param settings The index settings (ie. number of shards) */ - void createIndex(String index, Integer version, Map settings=null) { + void createIndex(String index, Integer version, Map settings=null, Map esMappings = [:]) { index = versionIndex(index, version) - createIndex(index, settings) + createIndex(index, settings, esMappings) } /** diff --git a/src/integration-test/groovy/grails/plugins/elasticsearch/ElasticSearchServiceIntegrationSpec.groovy b/src/integration-test/groovy/grails/plugins/elasticsearch/ElasticSearchServiceIntegrationSpec.groovy index ca123e16..b2c09247 100644 --- a/src/integration-test/groovy/grails/plugins/elasticsearch/ElasticSearchServiceIntegrationSpec.groovy +++ b/src/integration-test/groovy/grails/plugins/elasticsearch/ElasticSearchServiceIntegrationSpec.groovy @@ -30,6 +30,8 @@ import spock.lang.Specification import test.* import test.custom.id.Toy +import java.math.RoundingMode + @Integration @Rollback class ElasticSearchServiceIntegrationSpec extends Specification { @@ -272,7 +274,7 @@ class ElasticSearchServiceIntegrationSpec extends Specification { when: 'a geo distance filter search is performed' Map params = [indices: Building, types: Building] - Closure query = QueryBuilders.matchAllQuery() + QueryBuilder query = QueryBuilders.matchAllQuery() def location = '50, 13' Closure filter = { @@ -592,10 +594,13 @@ class ElasticSearchServiceIntegrationSpec extends Specification { } def result = elasticSearchService.search(params, query, filter) - then: 'all geo points in the search radius are found' + and: List searchResults = result.searchResults + //Avoid double precission issues + def sortResults = result.sort.(searchResults[0].id).collect {(it as BigDecimal).setScale(4, RoundingMode.HALF_UP) } - result.sort.(searchResults[0].id) == [2.5382648464733575] + then: 'all geo points in the search radius are found' + sortResults == [2.5383] } void 'Component as an inner object'() { diff --git a/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy b/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy index 36c91a4a..60bf7f7b 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy @@ -16,9 +16,14 @@ package grails.plugins.elasticsearch +import org.elasticsearch.Version import org.elasticsearch.client.transport.TransportClient import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.transport.InetSocketTransportAddress +import org.elasticsearch.mapper.attachments.MapperAttachmentsPlugin +import org.elasticsearch.node.Node +import org.elasticsearch.node.internal.InternalSettingsPreparer +import org.elasticsearch.plugins.Plugin import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.FactoryBean @@ -46,25 +51,25 @@ class ClientNodeFactoryBean implements FactoryBean { if (!(clientMode in SUPPORTED_MODES)) { throw new IllegalArgumentException("Invalid client mode, expected values were ${SUPPORTED_MODES}.") } - def nb = nodeBuilder() + Settings.Builder settings = Settings.settingsBuilder() def configFile = elasticSearchContextHolder.config.bootstrap.config.file if (configFile) { LOG.info "Looking for bootstrap configuration file at: $configFile" Resource resource = new PathMatchingResourcePatternResolver().getResource(configFile) - nb.settings(Settings.settingsBuilder().loadFromStream(configFile, resource.inputStream)) + settings = settings.loadFromStream(configFile, resource.inputStream) } def transportClient // Cluster name if (elasticSearchContextHolder.config.cluster.name) { - nb.clusterName(elasticSearchContextHolder.config.cluster.name) + settings.put('cluster.name', elasticSearchContextHolder.config.cluster.name) } // Path to the data folder of ES def dataPath = elasticSearchContextHolder.config.path.data if (dataPath) { - nb.settings.put('path.data', dataPath as String) + settings.put('path.data', dataPath as String) LOG.info "Using ElasticSearch data path: ${dataPath}" } @@ -121,14 +126,14 @@ class ClientNodeFactoryBean implements FactoryBean { // Determines how the data is stored (on disk, in memory, ...) def storeType = elasticSearchContextHolder.config.index.store.type if (storeType) { - nb.settings().put('index.store.type', storeType as String) + settings.put('index.store.type', storeType as String) LOG.debug "Local ElasticSearch client with store type of ${storeType} configured." } else { LOG.debug "Local ElasticSearch client with default store type configured." } def gatewayType = elasticSearchContextHolder.config.gateway.type if (gatewayType) { - nb.settings().put('gateway.type', gatewayType as String) + settings.put('gateway.type', gatewayType as String) LOG.debug "Local ElasticSearch client with gateway type of ${gatewayType} configured." } else { LOG.debug "Local ElasticSearch client with default gateway type configured." @@ -136,32 +141,32 @@ class ClientNodeFactoryBean implements FactoryBean { def queryParsers = elasticSearchContextHolder.config.index.queryparser if (queryParsers) { queryParsers.each { type, clz -> - nb.settings().put("index.queryparser.types.${type}".toString(), clz) + settings.put("index.queryparser.types.${type}".toString(), clz) } } def pluginsDirectory = elasticSearchContextHolder.config.path.plugins if (pluginsDirectory) { - nb.settings().put('path.plugins', pluginsDirectory as String) + settings.put('path.plugins', new File(pluginsDirectory as String).absolutePath) } // Path to the config folder of ES def confDirectory = elasticSearchContextHolder.config.path.conf if (confDirectory) { - nb.settings().put('path.conf', confDirectory as String) + settings.put('path.conf', confDirectory as String) } def tmpDirectory = tmpDirectory() LOG.info "Setting embedded ElasticSearch tmp dir to ${tmpDirectory}" - nb.settings().put("path.home", tmpDirectory) + settings.put("path.home", tmpDirectory) - nb.local(true) + settings.put("node.local", true) break case 'dataNode': def storeType = elasticSearchContextHolder.config.index.store.type if (storeType) { - nb.settings().put('index.store.type', storeType as String) + settings.put('index.store.type', storeType as String) LOG.debug "DataNode ElasticSearch client with store type of ${storeType} configured." } else { LOG.debug "DataNode ElasticSearch client with default store type configured." @@ -169,20 +174,20 @@ class ClientNodeFactoryBean implements FactoryBean { def queryParsers = elasticSearchContextHolder.config.index.queryparser if (queryParsers) { queryParsers.each { type, clz -> - nb.settings().put("index.queryparser.types.${type}".toString(), clz) + settings.put("index.queryparser.types.${type}".toString(), clz) } } if (elasticSearchContextHolder.config.discovery.zen.ping.unicast.hosts) { - nb.settings().put("discovery.zen.ping.unicast.hosts", elasticSearchContextHolder.config.discovery.zen.ping.unicast.hosts) + settings.put("discovery.zen.ping.unicast.hosts", elasticSearchContextHolder.config.discovery.zen.ping.unicast.hosts) } - nb.client(false) - nb.data(true) + settings.put("node.client", false) + settings.put("node.data", true) break case 'node': default: - nb.client(true) + settings.put("node.client", true) break } if (transportClient) { @@ -197,7 +202,7 @@ class ClientNodeFactoryBean implements FactoryBean { } // Avoiding this: - node = nb.node() + node = new PluginEnabledNode(settings, MapperAttachmentsPlugin) node.start() def client = node.client() // Wait for the cluster to become alive. @@ -232,4 +237,10 @@ class ClientNodeFactoryBean implements FactoryBean { file.deleteOnExit() return file.absolutePath } + + private static class PluginEnabledNode extends Node { + PluginEnabledNode(Settings.Builder settings, Class ... plugins) { + super(InternalSettingsPreparer.prepareEnvironment(settings.build(), null), Version.CURRENT, plugins as List) + } + } } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy index 2c61e73f..0d981e34 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy @@ -8,6 +8,8 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import static grails.plugins.elasticsearch.mapping.MappingMigrationStrategy.* +import static grails.plugins.elasticsearch.util.IndexNamingUtils.indexingIndexFor +import static grails.plugins.elasticsearch.util.IndexNamingUtils.queryingIndexFor /** * Created by @marcos-carceles on 26/01/15. @@ -43,79 +45,55 @@ class MappingMigrationManager { } def applyDeleteIndexStrategy(Map elasticMappings, List mappingConflicts, Map indexSettings) { - List deletedIndices = [] - mappingConflicts.each { - SearchableClassMapping scm = it.scm - if(!deletedIndices.contains(scm.indexName)) { - deletedIndices << scm.indexName - es.deleteIndex scm.indexName - int nextVersion = es.getNextVersion(scm.indexName) - es.createIndex scm.indexName, nextVersion, indexSettings - es.waitForIndex scm.indexName, nextVersion //Ensure it exists so later on mappings are created on the right version - es.pointAliasTo scm.indexName, scm.indexName, nextVersion - es.pointAliasTo scm.indexingIndex, scm.indexName, nextVersion - if(!esConfig.bulkIndexOnStartup) { //Otherwise, it will be done post content creation - if (!esConfig.migration.disableAliasChange) { - es.pointAliasTo scm.queryingIndex, scm.indexName, nextVersion - } - } - } + List indices = mappingConflicts.collect { it.scm.indexName } as Set + indices.each { String indexName -> + + es.deleteIndex indexName + + int nextVersion = es.getNextVersion(indexName) + boolean buildQueryingAlias = (!!esConfig.bulkIndexOnStartup) && (!esConfig.migration.disableAliasChange) + + rebuildIndexWithMappings(indexName, nextVersion, indexSettings, elasticMappings, buildQueryingAlias) } - rebuildMappings(elasticMappings, deletedIndices) + indices } def applyAliasStrategy(Map elasticMappings, List mappingConflicts, Map indexSettings) { - def migratedIndices = buildNextIndexVersion(mappingConflicts, indexSettings) - rebuildMappings(elasticMappings, migratedIndices) - } - private List buildNextIndexVersion(List conflictingMappings, Map indexSettings) { - def migratedIndices = [] - conflictingMappings.each { - SearchableClassMapping scm = it.scm - if (!migratedIndices.contains(scm.indexName)) { - migratedIndices << scm.indexName - println "index: $scm.indexName" - LOG.debug("Creating new version and alias for conflicting mapping ${scm.indexName}/${scm.elasticTypeName}") - boolean conflictOnAlias = es.aliasExists(scm.indexName) - println "Conflict: $conflictOnAlias" - println "Config: $esConfig.migration.aliasReplacesIndex" - if (conflictOnAlias || esConfig.migration.aliasReplacesIndex) { - int nextVersion = es.getNextVersion(scm.indexName) - if (!conflictOnAlias) { - es.deleteIndex(scm.indexName) - } - es.createIndex scm.indexName, nextVersion, indexSettings - es.waitForIndex scm.indexName, nextVersion - //Ensure it exists so later on mappings are created on the right version - es.pointAliasTo scm.indexName, scm.indexName, nextVersion - es.pointAliasTo scm.indexingIndex, scm.indexName, nextVersion - - if (!esConfig.bulkIndexOnStartup) { //Otherwise, it will be done post content creation - if (!conflictOnAlias || !esConfig.migration.disableAliasChange) { - es.pointAliasTo scm.queryingIndex, scm.indexName, nextVersion - } - } - } else { - throw new MappingException("Could not create alias ${scm.indexName} to solve error installing mapping ${scm.elasticTypeName}, index with the same name already exists.", it.exception) + List indices = mappingConflicts.collect { it.scm.indexName } as Set + + indices.each { String indexName -> + LOG.debug("Creating new version and alias for conflicting index ${indexName}") + boolean conflictOnAlias = es.aliasExists(indexName) + if (conflictOnAlias || esConfig.migration.aliasReplacesIndex) { + + if (!conflictOnAlias) { + es.deleteIndex(indexName) } + + int nextVersion = es.getNextVersion(indexName) + boolean buildQueryingAlias = (!esConfig.bulkIndexOnStartup) && (!conflictOnAlias || !esConfig.migration.disableAliasChange) + rebuildIndexWithMappings(indexName, nextVersion, indexSettings, elasticMappings, buildQueryingAlias) + + } else { + throw new MappingException("Could not create alias ${indexName} to solve error installing mappings, index with the same name already exists.") } } - migratedIndices + indices } - private void rebuildMappings(Map elasticMappings, List migratedIndices) { - //Recreate the mappings for all the indexes that were changed - elasticMappings.each { SearchableClassMapping scm, elasticMapping -> - if (migratedIndices.contains(scm.indexName)) { - elasticSearchContextHolder.deletedOnMigration << scm.domainClass.clazz - //Mark it for potential content index on Bootstrap - if (scm.isRoot()) { - int newVersion = es.getLatestVersion(scm.indexName) - String indexName = es.versionIndex(scm.indexName, newVersion) - es.createMapping(indexName, scm.elasticTypeName, elasticMapping) - } - } + private void rebuildIndexWithMappings(String indexName, int nextVersion, Map indexSettings, Map elasticMappings, boolean buildQueryingAlias) { + Map esMappings = elasticMappings.findAll { SearchableClassMapping scm, Map esMapping -> + scm.indexName == indexName && scm.isRoot() + }.collectEntries { SearchableClassMapping scm, Map esMapping -> + [(scm.elasticTypeName) : esMapping] + } + es.createIndex indexName, nextVersion, indexSettings, esMappings + es.waitForIndex indexName, nextVersion //Ensure it exists so later on mappings are created on the right version + es.pointAliasTo indexName, indexName, nextVersion + es.pointAliasTo indexingIndexFor(indexName), indexName, nextVersion + if (buildQueryingAlias) { + es.pointAliasTo queryingIndexFor(indexName), indexName, nextVersion } } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy index 55ba9762..af5c916e 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy @@ -16,6 +16,7 @@ package grails.plugins.elasticsearch.mapping +import grails.plugins.elasticsearch.util.IndexNamingUtils import grails.util.GrailsNameUtils import grails.core.GrailsDomainClass import grails.plugins.elasticsearch.ElasticSearchContextHolder @@ -30,9 +31,6 @@ class SearchableClassMapping { private boolean root = true protected all = true - public static final READ_SUFFIX = "_read" - public static final WRITE_SUFFIX = "_write" - String indexName SearchableClassMapping(GrailsDomainClass domainClass, Collection propertiesMapping) { @@ -91,11 +89,11 @@ class SearchableClassMapping { } String getIndexingIndex() { - return indexName + WRITE_SUFFIX + return IndexNamingUtils.indexingIndexFor(indexName) } String getQueryingIndex() { - return indexName + READ_SUFFIX + return IndexNamingUtils.queryingIndexFor(indexName) } /** diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy index 3c9f8d22..a3854795 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy @@ -29,6 +29,8 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import static grails.plugins.elasticsearch.mapping.MappingMigrationStrategy.* +import static grails.plugins.elasticsearch.util.IndexNamingUtils.indexingIndexFor +import static grails.plugins.elasticsearch.util.IndexNamingUtils.queryingIndexFor /** * Build searchable mappings, configure ElasticSearch indexes, @@ -99,33 +101,42 @@ class SearchableClassMappingConfigurator { LOG.debug "elasticMappings are ${elasticMappings.keySet()}" MappingMigrationStrategy migrationStrategy = esConfig?.migration?.strategy ? MappingMigrationStrategy.valueOf(esConfig.migration.strategy) : none - Set installedIndices = [] def mappingConflicts = [] - for (SearchableClassMapping scm : mappings) { - if (scm.isRoot()) { - Map elasticMapping = elasticMappings[scm] + Set indices = mappings.collect { it.indexName } as Set - // todo wait for success, maybe retry. - // If the index was not created, create it - if (!installedIndices.contains(scm.indexName)) { - try { - createIndexWithReadAndWrite(migrationStrategy, scm, indexSettings) - installedIndices.add(scm.indexName) - } catch (RemoteTransportException rte) { - LOG.debug(rte.getMessage()) - } - } + //Install the mappings for each index all together + indices.each { String indexName -> - // Install mapping - if (LOG.isDebugEnabled()) { - LOG.debug("Installing mapping [" + scm.elasticTypeName + "] => " + elasticMapping) - } + List indexMappings = mappings.findAll { it.indexName == indexName && it.isRoot() } + Map esMappings = indexMappings.collectEntries { [(it.elasticTypeName) : elasticMappings[it]] } + + //If the index does not exist we attempt to create all the mappings at once with it + if(!es.indexExists(indexName)) { try { - es.createMapping scm.indexName, scm.elasticTypeName, elasticMapping - } catch (MergeMappingException e) { - LOG.warn("Could not install mapping ${scm.indexName}/${scm.elasticTypeName} due to ${e.message}, migrations needed") - mappingConflicts << new MappingConflict(scm: scm, exception: e) + if (LOG.isDebugEnabled()) { + LOG.debug("Creating index [" + indexName + "] => with new mappings:") + esMappings.each {String type, Map esMapping -> + LOG.debug("\t\tMapping ["+ type +"] => " + esMapping) + } + } + createIndexWithMappings(indexName, migrationStrategy, esMappings, indexSettings) + } catch (RemoteTransportException rte) { + LOG.debug(rte.getMessage()) + } + } else { //We install the mappings one by one + indexMappings.each { SearchableClassMapping scm -> + Map elasticMapping = elasticMappings[scm] + // Install mapping + if (LOG.isDebugEnabled()) { + LOG.debug("Installing mapping [" + scm.elasticTypeName + "] => " + elasticMapping) + } + try { + es.createMapping scm.indexName, scm.elasticTypeName, elasticMapping + } catch (IllegalArgumentException|MergeMappingException e) { + LOG.warn("Could not install mapping ${scm.indexName}/${scm.elasticTypeName} due to ${e.message}, migrations needed") + mappingConflicts << new MappingConflict(scm: scm, exception: e) + } } } } @@ -144,23 +155,26 @@ class SearchableClassMappingConfigurator { * @returns true if it created a new index, false if it already existed * @throws RemoteTransportException if some other error occured */ - private boolean createIndexWithReadAndWrite(MappingMigrationStrategy strategy, SearchableClassMapping scm, Map indexSettings) throws RemoteTransportException { - // Could be blocked on index level, thus wait. + private boolean createIndexWithMappings(String indexName, MappingMigrationStrategy strategy, Map esMappings, Map indexSettings) throws RemoteTransportException { + // Could be blocked on cluster level, thus wait. + String queryingIndex = queryingIndexFor(indexName) + String indexingIndex = indexingIndexFor(indexName) + es.waitForClusterStatus(ClusterHealthStatus.YELLOW) - if(!es.indexExists(scm.indexName)) { - LOG.debug("Index ${scm.indexName} does not exists, initiating creation...") + if(!es.indexExists(indexName)) { + LOG.debug("Index ${indexName} does not exists, initiating creation...") if (strategy == alias) { - def nextVersion = es.getNextVersion scm.indexName - es.createIndex scm.indexName, nextVersion, indexSettings - es.pointAliasTo scm.indexName, scm.indexName, nextVersion + def nextVersion = es.getNextVersion indexName + es.createIndex indexName, nextVersion, indexSettings, esMappings + es.pointAliasTo indexName, indexName, nextVersion } else { - es.createIndex scm.indexName, indexSettings + es.createIndex indexName, indexSettings } } //Create them only if they don't exist so it does not mess with other migrations - if(!es.aliasExists(scm.queryingIndex)) { - es.pointAliasTo(scm.queryingIndex, scm.indexName) - es.pointAliasTo(scm.indexingIndex, scm.indexName) + if(!es.aliasExists(queryingIndex)) { + es.pointAliasTo(queryingIndex, indexName) + es.pointAliasTo(indexingIndex, indexName) } } From 089419965a684ba95c3c4a0d22bef8a0d54c1e22 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Wed, 1 Jun 2016 16:25:01 +0200 Subject: [PATCH 06/16] Missing file --- .../elasticsearch/util/IndexNamingUtils.groovy | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/main/groovy/grails/plugins/elasticsearch/util/IndexNamingUtils.groovy diff --git a/src/main/groovy/grails/plugins/elasticsearch/util/IndexNamingUtils.groovy b/src/main/groovy/grails/plugins/elasticsearch/util/IndexNamingUtils.groovy new file mode 100644 index 00000000..041e2e85 --- /dev/null +++ b/src/main/groovy/grails/plugins/elasticsearch/util/IndexNamingUtils.groovy @@ -0,0 +1,18 @@ +package grails.plugins.elasticsearch.util + +/** + * Created by marcoscarceles on 16/05/2016. + */ +class IndexNamingUtils { + + public static final READ_SUFFIX = "_read" + public static final WRITE_SUFFIX = "_write" + + static String queryingIndexFor(String indexName) { + indexName + READ_SUFFIX + } + + static String indexingIndexFor(String indexName) { + indexName + WRITE_SUFFIX + } +} From 6d3c5170dc2466cb5666447fa487e7a4c03f9f84 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Wed, 1 Jun 2016 16:27:55 +0200 Subject: [PATCH 07/16] git ignored Elasticsearch plugin binaries --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5966c4cd..7775e4cc 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ plugin.xml .project test/reports data/ + +#Elasticserch plugin binaries +src/integration-test/resources/elasticsearch/plugins \ No newline at end of file From f58b9cd086678edea6728b8feb55920b64805257 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Thu, 2 Jun 2016 08:42:48 +0200 Subject: [PATCH 08/16] Fix test using old READ and WRITE suffixes --- .../mapping/SearchableClassMappingSpec.groovy | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/test/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingSpec.groovy b/src/test/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingSpec.groovy index 2e41f634..78ba52df 100644 --- a/src/test/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingSpec.groovy +++ b/src/test/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingSpec.groovy @@ -2,15 +2,13 @@ package grails.plugins.elasticsearch.mapping import grails.core.GrailsApplication import grails.core.GrailsDomainClass +import grails.plugins.elasticsearch.util.IndexNamingUtils import grails.test.mixin.Mock import org.grails.core.DefaultGrailsDomainClass import spock.lang.Specification import test.Photo import test.upperCase.UpperCase -import static grails.plugins.elasticsearch.mapping.SearchableClassMapping.READ_SUFFIX -import static grails.plugins.elasticsearch.mapping.SearchableClassMapping.WRITE_SUFFIX - @Mock([Photo, UpperCase]) class SearchableClassMappingSpec extends Specification { @@ -24,8 +22,8 @@ class SearchableClassMappingSpec extends Specification { then: scm.indexName == packageName - scm.queryingIndex == packageName + READ_SUFFIX - scm.indexingIndex == packageName + WRITE_SUFFIX + scm.queryingIndex == IndexNamingUtils.queryingIndexFor(packageName) + scm.indexingIndex == IndexNamingUtils.indexingIndexFor(packageName) scm.queryingIndex != scm.indexingIndex scm.indexName != scm.queryingIndex scm.indexName != scm.indexingIndex From dbf8c34f381816d65a1c0a750728b3c10a3fc08b Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Thu, 2 Jun 2016 10:39:28 +0200 Subject: [PATCH 09/16] Compile static (which helps to identify errors) --- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.jar | Bin 53636 -> 53638 bytes gradlew.bat | 2 +- .../ElasticSearchBootStrapHelper.groovy | 35 ++++-------- .../ElasticSearchMappingFactory.groovy | 20 +++---- .../mapping/MappingConflict.groovy | 3 ++ .../mapping/MappingMigrationManager.groovy | 49 ++++++----------- .../mapping/MappingMigrationStrategy.groovy | 3 ++ .../mapping/SearchableClassMapping.groovy | 14 +++-- .../SearchableClassMappingConfigurator.groovy | 51 ++++++------------ .../SearchableClassPropertyMapping.groovy | 8 +-- .../SearchableDomainClassMapper.groovy | 8 +-- 12 files changed, 81 insertions(+), 114 deletions(-) diff --git a/gradle.properties b/gradle.properties index 69156480..d388b8bc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ -grailsVersion=3.1.8 +grailsVersion=3.1.1 gradleWrapperVersion=2.9 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 941144813d241db74e1bf25b6804c679fbe7f0a3..5ccda13e9cb94678ba179b32452cf3d60dc36353 100644 GIT binary patch delta 2517 zcmaJ?3s6*57(Vyz!fvpv@>qf^k983giC8fo35+-xkgK=|ge3_m2%|edh>yhNkzhki z1jn?*LV`q%;k76}Fi{>0$a54%EmV+@1aY-Vg|IsJoGZ+=raN=zoc;Fu{{Q>W`Omrg ziv*cP0;yUh5E=m>76ZoL-#=ICXu^&$wa??d0RSPn-a(eu%#*~&i4>To&S0r0LM51d1~#B9K-lHZvrci}VJEiA>!L2)4{Jhr`;fLPP{} zl82a+S;|6Vj`zhvn^DpB=m(?qmD3y)!65 zwPu~NLanT*c2`!oFIFe)GznC?E8n^Y{UV%A1c;t@Yw?0kv<^1#lL-41ubdU}^>RtZ zqsgyMimw^w2@cVcodbvBJL0Srva&4r*MiTl(!iZ9OKa=?!IT zg}WnNgzQ9H5_y0&WoPc;(K*La`NQa)2fLHzz|B!Lgku-|7fojJ*kNua*_KCP-SXFC z|JI2BIA9Kd7uH#{+GrnXxVfx}b z-0WJ(X(9PHyOb7S_h;%VwttNRo-!ov=R>B`QhTTTdrp#36~?*wRqM@(ozHf1*yOn& zvUt8{#Lv700JOdafCu)K({2DI-7@WWwSEFITeRkA;6eEC((?c4MrF`+H@;gpTgaYD z7+1HDe>`DX-J%q>uh9kpWEnp~ZUkT^c(EcM{fbYFaZ{@(+Co5%`l||Kp5hXxh)Il$ z(nSBS5bZDE#B8nyFjr{##;ciP)1$T&Az}}D9nkyosZO({U|ORLmW7I8+hq$mYpd~g zMy~~`Bmq^i@+A|E)oBzxMYg@N$etzNA6>uw6#DaL^eTAdUGSqT3ou7`a!+3v3K)n2 zy7D-Y@bVQ;d}lzao}tBJ9Dk`;4%atGu$eZKjlQ`^JB~VmueFmRvJE^^Z%+bTL;)sd z;s9gy&IFs)z|gLi82ivtG8MEVaU@HI4o93604JWIIx9@pTbaYs1_=pLi-K6oG5)Z@ znqWqa44t?TV;+r8xJv)tnmO0d<~mrRlgbO422&fC5Zwi&JK}?Nqm2%@!kZnYyN{vj zs*xG~3LXsLl8T_JiR{f*`PMJ1DFEI>UA>f-Bq;=w^;RaZuxSpCoVR(^ENd|Usu=*x z=8^Y>Vscri6pD3Zq1kI$jYt9L<+n(0ID8!L2;bL9?Aa4Dbs-}gsMM*aIxPn7*UiNz z8J6o9N&{PUcI4-%j-m4-VSkA=?1*HbN3(?Bs%D0Eih`Bd4lt|PoM43rGmFO9-DWPF zaSKCR#KOlN&QRT=kh3<19Za71Aqy(h91EsQuw4qaA+ZfCYjMDSc{d@CcnIoSR+FUO ztqgrO2`Bh7X(k+PBef20>Y3Y*62zfqS0+KW9a%U*q7zWt{zUzyLEZy4rimp z{%jQ_;3N{ilY=k?T-MG7RJSvWD*UFAG+Ce(MlT>egm{fVGG^P-zvrw|kPF-b}K4-+ue;xBve8UtXEf zt}-KWNPrQ40sulGNb}j~S1g`k#9e0E8F&gjue5n)W1(2Mo*vhFK}&;$OJLt=X%$~E#C9XpN2zi0vMHz+(J zQ!~s$C|B-OVCAjlo zx<|mO)8$n)32tqJ3Cdfg1;L??Q8w8=wc$bL&kh#-`B;4O-@LW0w|n-sE-S7-nR0$0 zsNrEE)iE+`{fX(lS5)8ORnf(tcRTup`I~w-KIy1c?l7D?vsJh?b{3!Ms4HXlP`C#@ z;`#|Biyx!J%wS`6prV)pV5c$sre=SO{c!YZ_#ZE|ix=nUQq}QLhfbBVD zT{I@xi=0t*#%S(k6N2^T_pn%bVHH_%ce!Rtq67e)E0kx+;Ls+4dS_9lgz(3X4p9>s zUw*v_k4liQBgOzI;n5mbwM%&gYpw72PkciEvYKLTl-4|ENB7i6vr9fu`>?!Or@C2e z>~t8*i)bt{5;9Baox`TX$7dG5l|uCj&k#cj1vF3NGdB*VHY>ST57st&zRM&dN-Tgp zED1$58M3~%Zwt6{}oujBuaFuPvm>evoVNI(D7X^5xm8M25Fh11k zLNMDlnwl-eSWuf4!IBYn#tMbb0ghP-;E6VNeiOp2WN2$PaHLH_xFRi0ZB*je2J(=Q?Sj#dE77#E^I!`0}-Y*?F3hBw;+K-2|>klET4G5fHL6EBtOXPJV}~sXZ&wFR4dlWj)ecb9DZzUvSmK$o4GEYcfdz*7XtkDn zcq@OQ;cW6dY3e{B7CqOgAcA8E^DRQyOt`n3I9g-weX$Jf)dlU9djSWJ;>r6j*yEKs zC784FonzzHUCKqX{$JFXjM(CICp-q#0@@S+FU~|Qbnp6rINRPuQ`yyMt8U}5YG|%a z{|Y&)LdV5D*IhN^3)*x$V0X@Ze~~qcOCqxs&LWxhh-?Ez1?Eg_Fxj_PdjBB`;5Oul zo1<6NSov!AOng@$@H)gq`vp>+&rzKU?Rxx)_2eF!8a$2Xx&16*@`B*)2!Fp3m0TK diff --git a/gradlew.bat b/gradlew.bat index 8a0b282a..5f192121 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -46,7 +46,7 @@ echo location of your Java installation. goto fail :init -@rem Get command-line arguments, handling Windowz variants +@rem Get command-line arguments, handling Windows variants if not "%OS%" == "Windows_NT" goto win9xME_args if "%@eval[2+2]" == "4" goto 4NT_args diff --git a/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchBootStrapHelper.groovy b/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchBootStrapHelper.groovy index 63d73c6a..174c1493 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchBootStrapHelper.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchBootStrapHelper.groovy @@ -3,6 +3,8 @@ package grails.plugins.elasticsearch import grails.core.GrailsApplication import grails.plugins.elasticsearch.mapping.MappingMigrationStrategy import grails.plugins.elasticsearch.mapping.SearchableClassMapping +import grails.plugins.elasticsearch.util.ElasticSearchConfigAware +import groovy.transform.CompileStatic import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -13,17 +15,17 @@ import static MappingMigrationStrategy.none * Created by @marcos-carceles on 13/01/15. * Created and exposed as a bean, because Bootstrap cannot be easily tested and invoked from IntegrationSpec */ -class ElasticSearchBootStrapHelper { +@CompileStatic +class ElasticSearchBootStrapHelper implements ElasticSearchConfigAware { private static final Logger LOG = LoggerFactory.getLogger(this) - private GrailsApplication grailsApplication - private ElasticSearchService elasticSearchService - private ElasticSearchAdminService elasticSearchAdminService - private ElasticSearchContextHolder elasticSearchContextHolder + GrailsApplication grailsApplication + ElasticSearchService elasticSearchService + ElasticSearchAdminService elasticSearchAdminService + ElasticSearchContextHolder elasticSearchContextHolder void bulkIndexOnStartup() { - def esConfig = grailsApplication.config.elasticSearch def bulkIndexOnStartup = esConfig?.bulkIndexOnStartup //Index Content if (bulkIndexOnStartup == "deleted") { //Index lost content due to migration @@ -34,33 +36,16 @@ class ElasticSearchBootStrapHelper { elasticSearchService.index() } //Update index aliases where needed - MappingMigrationStrategy migrationStrategy = esConfig?.migration?.strategy ? MappingMigrationStrategy.valueOf(esConfig.migration.strategy) : none + MappingMigrationStrategy migrationStrategy = migrationConfig?.strategy ? MappingMigrationStrategy.valueOf(migrationConfig?.strategy as String) : none if (migrationStrategy == alias) { elasticSearchContextHolder.deletedOnMigration.each { Class clazz -> SearchableClassMapping scm = elasticSearchContextHolder.getMappingContextByType(clazz) int latestVersion = elasticSearchAdminService.getLatestVersion(scm.indexName) - if(!esConfig.migration.disableAliasChange) { + if(!migrationConfig?.disableAliasChange) { elasticSearchAdminService.pointAliasTo scm.queryingIndex, scm.indexName, latestVersion } elasticSearchAdminService.pointAliasTo scm.indexingIndex, scm.indexName, latestVersion } } } - - void setGrailsApplication(GrailsApplication grailsApplication) { - this.grailsApplication = grailsApplication - } - - void setElasticSearchService(ElasticSearchService elasticSearchService) { - this.elasticSearchService = elasticSearchService - } - - void setElasticSearchAdminService(ElasticSearchAdminService elasticSearchAdminService) { - this.elasticSearchAdminService = elasticSearchAdminService - } - - void setElasticSearchContextHolder(ElasticSearchContextHolder elasticSearchContextHolder) { - this.elasticSearchContextHolder = elasticSearchContextHolder - } - } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/ElasticSearchMappingFactory.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/ElasticSearchMappingFactory.groovy index f2bd8b91..9db4760b 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/ElasticSearchMappingFactory.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/ElasticSearchMappingFactory.groovy @@ -20,15 +20,17 @@ import grails.core.GrailsDomainClassProperty import grails.plugins.GrailsPluginManager import grails.util.GrailsNameUtils import grails.util.Holders +import groovy.transform.CompileStatic import org.springframework.util.ClassUtils /** * Build ElasticSearch class mapping based on attributes provided by closure. */ +@CompileStatic class ElasticSearchMappingFactory { private static final Set SUPPORTED_FORMAT = - ['string', 'integer', 'long', 'float', 'double', 'boolean', 'null', 'date'] + ['string', 'integer', 'long', 'float', 'double', 'boolean', 'null', 'date'] as Set private static Class JODA_TIME_BASE @@ -46,7 +48,7 @@ class ElasticSearchMappingFactory { Map mappingFields = [properties: getMappingProperties(scm)] if (scm.@all instanceof Map) { - mappingFields.'_all' = scm.@all + mappingFields.'_all' = scm.@all as Map } if (!scm.isAll()) mappingFields.'_all' = Collections.singletonMap('enabled', false) @@ -57,7 +59,7 @@ class ElasticSearchMappingFactory { } Map mapping = [:] - mapping."${scm.getElasticTypeName()}" = mappingFields + mapping.put("${scm.getElasticTypeName()}" as String, mappingFields) mapping } @@ -98,9 +100,9 @@ class ElasticSearchMappingFactory { idType = 'string' } - props.id = defaultDescriptor(idType, 'not_analyzed', true) - props.class = defaultDescriptor('string', 'no', true) - props.ref = defaultDescriptor('string', 'no', true) + props.put('id', defaultDescriptor(idType, 'not_analyzed', true)) + props.put('class', defaultDescriptor('string', 'no', true)) + props.put('ref', defaultDescriptor('string', 'no', true)) } } propOptions.type = propType @@ -122,8 +124,8 @@ class ElasticSearchMappingFactory { untouched.put('type', propOptions.get('type')) untouched.put('index', 'not_analyzed') - Map fields = [untouched: untouched] - fields."${scpm.getPropertyName()}" = field + Map fields = [untouched: untouched] + fields.put("${scpm.getPropertyName()}" as String, field) propOptions = [:] propOptions.type = 'multi_field' @@ -215,7 +217,7 @@ class ElasticSearchMappingFactory { } private static String treatValueAsAString(String idType) { - if (Holders.grailsApplication.config.elasticSearch.datastoreImpl =~ /mongo/) { + if ((Holders.grailsApplication.config.elasticSearch as ConfigObject).datastoreImpl =~ /mongo/) { idType = 'string' } else { def pluginManager = Holders.applicationContext.getBean(GrailsPluginManager.BEAN_NAME) diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingConflict.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingConflict.groovy index afeba079..195e7293 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingConflict.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingConflict.groovy @@ -1,8 +1,11 @@ package grails.plugins.elasticsearch.mapping +import groovy.transform.CompileStatic + /** * Created by @marcos-carceles on 26/01/15. */ +@CompileStatic class MappingConflict { SearchableClassMapping scm diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy index 0d981e34..14b46652 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy @@ -4,6 +4,8 @@ import grails.core.GrailsApplication import grails.plugins.elasticsearch.ElasticSearchAdminService import grails.plugins.elasticsearch.ElasticSearchContextHolder import grails.plugins.elasticsearch.exception.MappingException +import grails.plugins.elasticsearch.util.ElasticSearchConfigAware +import groovy.transform.CompileStatic import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -14,19 +16,15 @@ import static grails.plugins.elasticsearch.util.IndexNamingUtils.queryingIndexFo /** * Created by @marcos-carceles on 26/01/15. */ -class MappingMigrationManager { +@CompileStatic +class MappingMigrationManager implements ElasticSearchConfigAware { - private ElasticSearchContextHolder elasticSearchContextHolder - private ElasticSearchAdminService es - private GrailsApplication grailsApplication - private ConfigObject config + ElasticSearchContextHolder elasticSearchContextHolder + ElasticSearchAdminService es + GrailsApplication grailsApplication private static final Logger LOG = LoggerFactory.getLogger(this) - private Map getEsConfig() { - grailsApplication.config.elasticSearch as Map - } - def applyMigrations(MappingMigrationStrategy migrationStrategy, Map elasticMappings, List mappingConflicts, Map indexSettings) { switch (migrationStrategy) { case delete: @@ -44,35 +42,35 @@ class MappingMigrationManager { } } - def applyDeleteIndexStrategy(Map elasticMappings, List mappingConflicts, Map indexSettings) { - List indices = mappingConflicts.collect { it.scm.indexName } as Set + Set applyDeleteIndexStrategy(Map elasticMappings, List mappingConflicts, Map indexSettings) { + Set indices = mappingConflicts.collect { it.scm.indexName } as Set indices.each { String indexName -> es.deleteIndex indexName int nextVersion = es.getNextVersion(indexName) - boolean buildQueryingAlias = (!!esConfig.bulkIndexOnStartup) && (!esConfig.migration.disableAliasChange) + boolean buildQueryingAlias = (!!esConfig.bulkIndexOnStartup) && (migrationConfig?.disableAliasChange) rebuildIndexWithMappings(indexName, nextVersion, indexSettings, elasticMappings, buildQueryingAlias) } indices } - def applyAliasStrategy(Map elasticMappings, List mappingConflicts, Map indexSettings) { + Set applyAliasStrategy(Map elasticMappings, List mappingConflicts, Map indexSettings) { - List indices = mappingConflicts.collect { it.scm.indexName } as Set + Set indices = mappingConflicts.collect { it.scm.indexName } as Set indices.each { String indexName -> LOG.debug("Creating new version and alias for conflicting index ${indexName}") boolean conflictOnAlias = es.aliasExists(indexName) - if (conflictOnAlias || esConfig.migration.aliasReplacesIndex) { + if (conflictOnAlias || migrationConfig?.aliasReplacesIndex) { if (!conflictOnAlias) { es.deleteIndex(indexName) } int nextVersion = es.getNextVersion(indexName) - boolean buildQueryingAlias = (!esConfig.bulkIndexOnStartup) && (!conflictOnAlias || !esConfig.migration.disableAliasChange) + boolean buildQueryingAlias = (!esConfig.bulkIndexOnStartup) && (!conflictOnAlias || !migrationConfig?.disableAliasChange) rebuildIndexWithMappings(indexName, nextVersion, indexSettings, elasticMappings, buildQueryingAlias) } else { @@ -87,7 +85,7 @@ class MappingMigrationManager { scm.indexName == indexName && scm.isRoot() }.collectEntries { SearchableClassMapping scm, Map esMapping -> [(scm.elasticTypeName) : esMapping] - } + } as Map es.createIndex indexName, nextVersion, indexSettings, esMappings es.waitForIndex indexName, nextVersion //Ensure it exists so later on mappings are created on the right version es.pointAliasTo indexName, indexName, nextVersion @@ -96,21 +94,4 @@ class MappingMigrationManager { es.pointAliasTo queryingIndexFor(indexName), indexName, nextVersion } } - - void setElasticSearchContextHolder(ElasticSearchContextHolder elasticSearchContextHolder) { - this.elasticSearchContextHolder = elasticSearchContextHolder - } - - void setGrailsApplication(GrailsApplication grailsApplication) { - this.grailsApplication = grailsApplication - } - - void setConfig(ConfigObject config) { - this.config = config - } - - void setEs(ElasticSearchAdminService es) { - this.es = es - } - } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationStrategy.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationStrategy.groovy index aea63939..226e8d7c 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationStrategy.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationStrategy.groovy @@ -1,8 +1,11 @@ package grails.plugins.elasticsearch.mapping +import groovy.transform.CompileStatic + /** * Created by @marcos-carceles on 22/12/14. */ +@CompileStatic enum MappingMigrationStrategy { none, delete, deleteIndex, alias } \ No newline at end of file diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy index af5c916e..661c9d82 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy @@ -16,12 +16,16 @@ package grails.plugins.elasticsearch.mapping +import grails.core.GrailsApplication +import grails.plugins.elasticsearch.util.ElasticSearchConfigAware import grails.plugins.elasticsearch.util.IndexNamingUtils import grails.util.GrailsNameUtils import grails.core.GrailsDomainClass import grails.plugins.elasticsearch.ElasticSearchContextHolder +import groovy.transform.CompileStatic -class SearchableClassMapping { +@CompileStatic +class SearchableClassMapping implements ElasticSearchConfigAware { /** All searchable properties */ private Collection propertiesMapping @@ -80,7 +84,7 @@ class SearchableClassMapping { } String calculateIndexName() { - String name = domainClass.application?.config?.elasticSearch?.index?.name ?: domainClass.packageName + String name = esConfig['index.name'] ?: domainClass.packageName if (name == null || name.length() == 0) { // index name must be lowercase (org.elasticsearch.indices.InvalidIndexNameException) name = domainClass.getPropertyName() @@ -107,7 +111,7 @@ class SearchableClassMapping { if (all instanceof Boolean) { return all } else if (all instanceof Map) { - return all.enabled instanceof Boolean ? all.enabled : true + return (all as Map).enabled instanceof Boolean ? (all as Map).enabled : true } return true } @@ -117,4 +121,8 @@ class SearchableClassMapping { return "${getClass().name}(domainClass:$domainClass, propertiesMapping:$propertiesMapping, indexName:$indexName, isAll:${isAll()})" } + @Override + GrailsApplication getGrailsApplication() { + return domainClass.application + } } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy index a3854795..69297d1d 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy @@ -16,6 +16,8 @@ package grails.plugins.elasticsearch.mapping +import grails.plugins.elasticsearch.util.ElasticSearchConfigAware +import groovy.transform.CompileStatic import org.elasticsearch.cluster.health.ClusterHealthStatus import org.grails.core.artefact.DomainClassArtefactHandler import grails.core.GrailsApplication @@ -36,15 +38,15 @@ import static grails.plugins.elasticsearch.util.IndexNamingUtils.queryingIndexFo * Build searchable mappings, configure ElasticSearch indexes, * build and install ElasticSearch mappings. */ -class SearchableClassMappingConfigurator { +@CompileStatic +class SearchableClassMappingConfigurator implements ElasticSearchConfigAware { private static final Logger LOG = LoggerFactory.getLogger(this) - private ElasticSearchContextHolder elasticSearchContext - private GrailsApplication grailsApplication - private ElasticSearchAdminService es - private MappingMigrationManager mmm - private ConfigObject config + ElasticSearchContextHolder elasticSearchContext + GrailsApplication grailsApplication + ElasticSearchAdminService es + MappingMigrationManager mmm /** * Init method. @@ -56,7 +58,6 @@ class SearchableClassMappingConfigurator { public Collection mappings() { // TODO: Not able to reflect changes in config if we use instance field config in SearchableDomainClassMapper constructor - ConfigObject esConfig = grailsApplication.config.elasticSearch List mappings = [] for (GrailsClass clazz : grailsApplication.getArtefacts(DomainClassArtefactHandler.TYPE)) { GrailsDomainClass domainClass = (GrailsDomainClass) clazz @@ -91,8 +92,7 @@ class SearchableClassMappingConfigurator { * @param mappings searchable class mappings to be install. */ public void installMappings(Collection mappings){ - Map esConfig = grailsApplication.config.elasticSearch - Map indexSettings = buildIndexSettings(esConfig) + Map indexSettings = buildIndexSettings() LOG.debug("Index settings are " + indexSettings) @@ -100,15 +100,15 @@ class SearchableClassMappingConfigurator { Map elasticMappings = buildElasticMappings(mappings) LOG.debug "elasticMappings are ${elasticMappings.keySet()}" - MappingMigrationStrategy migrationStrategy = esConfig?.migration?.strategy ? MappingMigrationStrategy.valueOf(esConfig.migration.strategy) : none + MappingMigrationStrategy migrationStrategy = migrationConfig?.strategy ? MappingMigrationStrategy.valueOf(migrationConfig?.strategy as String) : none def mappingConflicts = [] - Set indices = mappings.collect { it.indexName } as Set + Set indices = mappings.collect { it.indexName } as Set //Install the mappings for each index all together indices.each { String indexName -> - List indexMappings = mappings.findAll { it.indexName == indexName && it.isRoot() } + List indexMappings = mappings.findAll { it.indexName == indexName && it.isRoot() } as List Map esMappings = indexMappings.collectEntries { [(it.elasticTypeName) : elasticMappings[it]] } //If the index does not exist we attempt to create all the mappings at once with it @@ -178,12 +178,12 @@ class SearchableClassMappingConfigurator { } } - private Map buildIndexSettings(Map esConfig) { + private Map buildIndexSettings() { Map indexSettings = new HashMap() indexSettings.put("number_of_replicas", numberOfReplicas()) // Look for default index settings. if (esConfig != null) { - Map indexDefaults = esConfig.get("index") + Map indexDefaults = esConfig.get("index") as Map LOG.debug("Retrieved index settings") if (indexDefaults != null) { for (Map.Entry entry : indexDefaults.entrySet()) { @@ -204,30 +204,11 @@ class SearchableClassMappingConfigurator { elasticMappings } - void setElasticSearchContext(ElasticSearchContextHolder elasticSearchContext) { - this.elasticSearchContext = elasticSearchContext - } - - void setGrailsApplication(GrailsApplication grailsApplication) { - this.grailsApplication = grailsApplication - } - - void setEs(ElasticSearchAdminService es) { - this.es = es - } - - void setMmm(MappingMigrationManager mmm) { - this.mmm = mmm - } - void setConfig(ConfigObject config) { - this.config = config - } - private int numberOfReplicas() { - def defaultNumber = elasticSearchContext.config.index.numberOfReplicas + def defaultNumber = (esConfig.index as ConfigObject).numberOfReplicas if (!defaultNumber) { return 0 } - defaultNumber + defaultNumber as int } } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassPropertyMapping.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassPropertyMapping.groovy index 42416020..1a157f39 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassPropertyMapping.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassPropertyMapping.groovy @@ -17,16 +17,18 @@ package grails.plugins.elasticsearch.mapping import grails.core.GrailsDomainClassProperty import grails.plugins.elasticsearch.ElasticSearchContextHolder +import groovy.transform.CompileStatic /** * Custom searchable property mapping. */ +@CompileStatic class SearchableClassPropertyMapping { - private static final Set SEARCHABLE_MAPPING_OPTIONS = ['boost', 'index', 'analyzer'] + private static final Set SEARCHABLE_MAPPING_OPTIONS = ['boost', 'index', 'analyzer'] as Set private static final Set SEARCHABLE_SPECIAL_MAPPING_OPTIONS = ['component', 'converter', 'reference', 'excludeFromAll', 'maxDepth', 'multi_field', 'parent', 'geoPoint', - 'alias', 'dynamic', 'attachment'] + 'alias', 'dynamic', 'attachment'] as Set /** Grails attributes of this property */ private GrailsDomainClassProperty grailsProperty @@ -114,7 +116,7 @@ class SearchableClassPropertyMapping { int getMaxDepth() { Object maxDepth = specialMappingAttributes.maxDepth - maxDepth != null ? maxDepth : 0 as int + (maxDepth != null ? maxDepth : 0) as int } Class getBestGuessReferenceType() { diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableDomainClassMapper.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableDomainClassMapper.groovy index cf0d89ca..ebc557d4 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableDomainClassMapper.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableDomainClassMapper.groovy @@ -20,6 +20,7 @@ import grails.core.GrailsApplication import grails.core.GrailsDomainClass import grails.core.GrailsDomainClassProperty import grails.util.GrailsClassUtils +import groovy.transform.CompileStatic import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory import org.grails.core.DefaultGrailsDomainClass @@ -28,6 +29,7 @@ import org.springframework.util.Assert import java.lang.reflect.Modifier +@CompileStatic class SearchableDomainClassMapper extends GroovyObjectSupport { /** * Class mapping properties @@ -144,7 +146,7 @@ class SearchableDomainClassMapper extends GroovyObjectSupport { buildHashMapMapping((LinkedHashMap) searchable, domainClass, inheritedProperties) } else if (searchable instanceof Closure) { Set inheritedProperties = getInheritedProperties(domainClass) - buildClosureMapping(domainClass, (Closure) searchable, inheritedProperties) + buildClosureMapping(domainClass, searchable as Closure, inheritedProperties) } else { throw new IllegalArgumentException("'$searchablePropertyName' property has unknown type: " + searchable.getClass()) } @@ -280,7 +282,7 @@ class SearchableDomainClassMapper extends GroovyObjectSupport { return Collections.singleton(arg) } if (arg instanceof Object[]) { - return new HashSet(Arrays.asList(arg)) + return new HashSet(Arrays.asList(arg as String[])) } if (arg instanceof Collection) { //noinspection unchecked @@ -290,7 +292,7 @@ class SearchableDomainClassMapper extends GroovyObjectSupport { } private String getSearchablePropertyName() { - def searchablePropertyName = esConfig.searchableProperty.name + def searchablePropertyName = (esConfig.searchableProperty as ConfigObject).name //Maintain backwards compatibility. Searchable property name may not be defined if (!searchablePropertyName) { From 7031f29199c52f305010584a3c120917d7ac79b6 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Thu, 2 Jun 2016 16:03:44 +0200 Subject: [PATCH 10/16] Building transport client using ES 2.3 API's --- .../grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy b/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy index 60bf7f7b..db6c1d36 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy @@ -34,8 +34,6 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import static org.elasticsearch.node.NodeBuilder.nodeBuilder - class ClientNodeFactoryBean implements FactoryBean { static final SUPPORTED_MODES = ['local', 'transport', 'node', 'dataNode'] @@ -90,7 +88,7 @@ class ClientNodeFactoryBean implements FactoryBean { if (elasticSearchContextHolder.config.cluster.name) { transportSettings.put('cluster.name', elasticSearchContextHolder.config.cluster.name.toString()) } - transportClient = new TransportClient(transportSettings) + transportClient = TransportClient.builder().settings(transportSettings).build() boolean ip4Enabled = elasticSearchContextHolder.config.shield.ip4Enabled ?: true boolean ip6Enabled = elasticSearchContextHolder.config.shield.ip6Enabled ?: false From 76e7c6b6b30a5db43a994bff07d3aff75a248bc0 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Thu, 2 Jun 2016 16:04:32 +0200 Subject: [PATCH 11/16] Fixing tests (WIP) --- .../ElasticSearchBootStrapHelper.groovy | 16 ++++++------ .../ElasticSearchContextHolder.groovy | 20 ++++++++++++--- .../ElasticsearchGrailsPlugin.groovy | 2 -- .../mapping/MappingMigrationManager.groovy | 8 +++--- .../SearchableClassMappingConfigurator.groovy | 25 ++++++++++--------- 5 files changed, 42 insertions(+), 29 deletions(-) diff --git a/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchBootStrapHelper.groovy b/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchBootStrapHelper.groovy index 174c1493..47c3669a 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchBootStrapHelper.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchBootStrapHelper.groovy @@ -2,8 +2,8 @@ package grails.plugins.elasticsearch import grails.core.GrailsApplication import grails.plugins.elasticsearch.mapping.MappingMigrationStrategy -import grails.plugins.elasticsearch.mapping.SearchableClassMapping import grails.plugins.elasticsearch.util.ElasticSearchConfigAware +import grails.plugins.elasticsearch.util.IndexNamingUtils import groovy.transform.CompileStatic import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -29,8 +29,9 @@ class ElasticSearchBootStrapHelper implements ElasticSearchConfigAware { def bulkIndexOnStartup = esConfig?.bulkIndexOnStartup //Index Content if (bulkIndexOnStartup == "deleted") { //Index lost content due to migration - LOG.debug "Performing bulk indexing of classes requiring index/mapping migration ${elasticSearchContextHolder.deletedOnMigration} on their new version." - elasticSearchService.index(elasticSearchContextHolder.deletedOnMigration as Class[]) + LOG.debug "Performing bulk indexing of classes requiring index/mapping migration ${elasticSearchContextHolder.indexesRebuiltOnMigration} on their new version." + Class[] domainsToReindex = elasticSearchContextHolder.findMappedClassesOnIndices(elasticSearchContextHolder.indexesRebuiltOnMigration) as Class[] + elasticSearchService.index(domainsToReindex) } else if (bulkIndexOnStartup) { //Index all LOG.debug "Performing bulk indexing." elasticSearchService.index() @@ -38,13 +39,12 @@ class ElasticSearchBootStrapHelper implements ElasticSearchConfigAware { //Update index aliases where needed MappingMigrationStrategy migrationStrategy = migrationConfig?.strategy ? MappingMigrationStrategy.valueOf(migrationConfig?.strategy as String) : none if (migrationStrategy == alias) { - elasticSearchContextHolder.deletedOnMigration.each { Class clazz -> - SearchableClassMapping scm = elasticSearchContextHolder.getMappingContextByType(clazz) - int latestVersion = elasticSearchAdminService.getLatestVersion(scm.indexName) + elasticSearchContextHolder.indexesRebuiltOnMigration.each { String indexName -> + int latestVersion = elasticSearchAdminService.getLatestVersion(indexName) if(!migrationConfig?.disableAliasChange) { - elasticSearchAdminService.pointAliasTo scm.queryingIndex, scm.indexName, latestVersion + elasticSearchAdminService.pointAliasTo IndexNamingUtils.queryingIndexFor(indexName), indexName, latestVersion } - elasticSearchAdminService.pointAliasTo scm.indexingIndex, scm.indexName, latestVersion + elasticSearchAdminService.pointAliasTo IndexNamingUtils.indexingIndexFor(indexName), indexName, latestVersion } } } diff --git a/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchContextHolder.groovy b/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchContextHolder.groovy index 6eba36b4..e2c95687 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchContextHolder.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/ElasticSearchContextHolder.groovy @@ -2,7 +2,9 @@ package grails.plugins.elasticsearch import grails.core.GrailsDomainClass import grails.plugins.elasticsearch.mapping.SearchableClassMapping +import groovy.transform.CompileStatic +@CompileStatic class ElasticSearchContextHolder { /** * The configuration of the ElasticSearch plugin @@ -15,9 +17,9 @@ class ElasticSearchContextHolder { Map mapping = [:] /** - * A Set containing all the mappings that were deleted and created again by the migration strategy + * A Set containing all the indices that were regenerated during migration */ - Set deletedOnMigration = [] as Set + Set indexesRebuiltOnMigration = [] as Set /** * Adds a mapping context to the current mapping holder @@ -63,7 +65,7 @@ class ElasticSearchContextHolder { * @return A boolean determining if the class is root-mapped or not */ boolean isRootClass(Class clazz) { - mapping.values().any { scm -> scm.domainClass.clazz == clazz && scm.root } + mapping.values().any { scm -> scm.domainClass.clazz == clazz && scm.isRoot() } } /** @@ -76,6 +78,18 @@ class ElasticSearchContextHolder { findMappingContextByElasticType(elasticTypeName)?.domainClass?.clazz } + /** + * Returns all the Classes associated to a specific elasticSearch index + * + * @param elasticTypeName + * @return A Class instance or NULL if the class was not found + */ + List findMappedClassesOnIndices(Set indices) { + mapping.values().findAll { SearchableClassMapping scm -> + scm.indexName in indices + }*.domainClass*.clazz as List + } + /** * Returns the SearchableClassMapping that is associated to a elasticSearch type * @param elasticTypeName diff --git a/src/main/groovy/grails/plugins/elasticsearch/ElasticsearchGrailsPlugin.groovy b/src/main/groovy/grails/plugins/elasticsearch/ElasticsearchGrailsPlugin.groovy index 611ea12e..dc93a812 100755 --- a/src/main/groovy/grails/plugins/elasticsearch/ElasticsearchGrailsPlugin.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/ElasticsearchGrailsPlugin.groovy @@ -88,7 +88,6 @@ class ElasticsearchGrailsPlugin extends Plugin { mappingMigrationManager(MappingMigrationManager) { elasticSearchContextHolder = ref('elasticSearchContextHolder') grailsApplication = grailsApplication - config = esConfig es = ref('elasticSearchAdminService') } searchableClassMappingConfigurator(SearchableClassMappingConfigurator) { bean -> @@ -96,7 +95,6 @@ class ElasticsearchGrailsPlugin extends Plugin { grailsApplication = grailsApplication es = ref('elasticSearchAdminService') mmm = ref('mappingMigrationManager') - config = esConfig bean.initMethod = 'configureAndInstallMappings' } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy index 14b46652..c4f40829 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/MappingMigrationManager.groovy @@ -32,9 +32,10 @@ class MappingMigrationManager implements ElasticSearchConfigAware { " To prevent data loss, this strategy has been replaced by 'deleteIndex'") throw new MappingException() case deleteIndex: - applyDeleteIndexStrategy(elasticMappings, mappingConflicts, indexSettings) + elasticSearchContextHolder.indexesRebuiltOnMigration = applyDeleteIndexStrategy(elasticMappings, mappingConflicts, indexSettings) + break; case alias: - applyAliasStrategy(elasticMappings, mappingConflicts, indexSettings) + elasticSearchContextHolder.indexesRebuiltOnMigration = applyAliasStrategy(elasticMappings, mappingConflicts, indexSettings) break; case none: LOG.error("Could not install mappings : ${mappingConflicts}. No migration strategy selected.") @@ -49,9 +50,8 @@ class MappingMigrationManager implements ElasticSearchConfigAware { es.deleteIndex indexName int nextVersion = es.getNextVersion(indexName) - boolean buildQueryingAlias = (!!esConfig.bulkIndexOnStartup) && (migrationConfig?.disableAliasChange) - rebuildIndexWithMappings(indexName, nextVersion, indexSettings, elasticMappings, buildQueryingAlias) + rebuildIndexWithMappings(indexName, nextVersion, indexSettings, elasticMappings, true) } indices } diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy index 69297d1d..571c4ab0 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy @@ -133,12 +133,22 @@ class SearchableClassMappingConfigurator implements ElasticSearchConfigAware { } try { es.createMapping scm.indexName, scm.elasticTypeName, elasticMapping - } catch (IllegalArgumentException|MergeMappingException e) { + } catch (IllegalArgumentException e) { + LOG.warn("Could not install mapping ${scm.indexName}/${scm.elasticTypeName} due to ${e.message}, migrations needed") + mappingConflicts << new MappingConflict(scm: scm, exception: e) + } catch (MergeMappingException e) { LOG.warn("Could not install mapping ${scm.indexName}/${scm.elasticTypeName} due to ${e.message}, migrations needed") mappingConflicts << new MappingConflict(scm: scm, exception: e) } } } + //Create them only if they don't exist so it does not mess with other migrations + String queryingIndex = queryingIndexFor(indexName) + String indexingIndex = indexingIndexFor(indexName) + if(!es.aliasExists(queryingIndex)) { + es.pointAliasTo(queryingIndex, indexName) + es.pointAliasTo(indexingIndex, indexName) + } } if(mappingConflicts) { LOG.info("Applying migrations ...") @@ -152,14 +162,10 @@ class SearchableClassMappingConfigurator implements ElasticSearchConfigAware { /** * Creates the Elasticsearch index once unblocked and its read and write aliases * @param indexName - * @returns true if it created a new index, false if it already existed * @throws RemoteTransportException if some other error occured */ - private boolean createIndexWithMappings(String indexName, MappingMigrationStrategy strategy, Map esMappings, Map indexSettings) throws RemoteTransportException { + private void createIndexWithMappings(String indexName, MappingMigrationStrategy strategy, Map esMappings, Map indexSettings) throws RemoteTransportException { // Could be blocked on cluster level, thus wait. - String queryingIndex = queryingIndexFor(indexName) - String indexingIndex = indexingIndexFor(indexName) - es.waitForClusterStatus(ClusterHealthStatus.YELLOW) if(!es.indexExists(indexName)) { LOG.debug("Index ${indexName} does not exists, initiating creation...") @@ -168,14 +174,9 @@ class SearchableClassMappingConfigurator implements ElasticSearchConfigAware { es.createIndex indexName, nextVersion, indexSettings, esMappings es.pointAliasTo indexName, indexName, nextVersion } else { - es.createIndex indexName, indexSettings + es.createIndex indexName, indexSettings, esMappings } } - //Create them only if they don't exist so it does not mess with other migrations - if(!es.aliasExists(queryingIndex)) { - es.pointAliasTo(queryingIndex, indexName) - es.pointAliasTo(indexingIndex, indexName) - } } private Map buildIndexSettings() { From 401d023148116b85250e0147af6777b2b9059c2a Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Thu, 2 Jun 2016 16:10:08 +0200 Subject: [PATCH 12/16] Missing file --- .../util/ElasticSearchConfigAware.groovy | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/main/groovy/grails/plugins/elasticsearch/util/ElasticSearchConfigAware.groovy diff --git a/src/main/groovy/grails/plugins/elasticsearch/util/ElasticSearchConfigAware.groovy b/src/main/groovy/grails/plugins/elasticsearch/util/ElasticSearchConfigAware.groovy new file mode 100644 index 00000000..643f5708 --- /dev/null +++ b/src/main/groovy/grails/plugins/elasticsearch/util/ElasticSearchConfigAware.groovy @@ -0,0 +1,21 @@ +package grails.plugins.elasticsearch.util + +import grails.core.GrailsApplication +import groovy.transform.CompileStatic + +/** + * Created by marcoscarceles on 02/06/2016. + */ +@CompileStatic +trait ElasticSearchConfigAware { + + abstract GrailsApplication getGrailsApplication() + + ConfigObject getEsConfig() { + grailsApplication.config.elasticSearch as ConfigObject + } + + ConfigObject getMigrationConfig() { + (grailsApplication.config.elasticSearch as ConfigObject).migration as ConfigObject + } +} \ No newline at end of file From 168d3a9ad552a4e58d99d3badec6408f1eced555 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Thu, 2 Jun 2016 16:35:09 +0200 Subject: [PATCH 13/16] Updated ES version on ext properties --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d1ad397f..c3c6634f 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ apply plugin: "org.grails.grails-gsp" ext { grailsVersion = project.grailsVersion gradleWrapperVersion = project.gradleWrapperVersion - elasticsearchVersion = "2.3.0" + elasticsearchVersion = "2.3.3" } sourceCompatibility = 1.8 From b9dccbd96096963e99ff1c1f7330d70f26c7fbdf Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Fri, 3 Jun 2016 12:50:46 +0200 Subject: [PATCH 14/16] Fix unit tests --- .../elasticsearch/mapping/SearchableClassMapping.groovy | 2 +- .../elasticsearch/util/ElasticSearchConfigAware.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy index 661c9d82..618ec0f7 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMapping.groovy @@ -84,7 +84,7 @@ class SearchableClassMapping implements ElasticSearchConfigAware { } String calculateIndexName() { - String name = esConfig['index.name'] ?: domainClass.packageName + String name = esConfig?.getProperty('index.name') ?: domainClass.packageName if (name == null || name.length() == 0) { // index name must be lowercase (org.elasticsearch.indices.InvalidIndexNameException) name = domainClass.getPropertyName() diff --git a/src/main/groovy/grails/plugins/elasticsearch/util/ElasticSearchConfigAware.groovy b/src/main/groovy/grails/plugins/elasticsearch/util/ElasticSearchConfigAware.groovy index 643f5708..53ceca6c 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/util/ElasticSearchConfigAware.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/util/ElasticSearchConfigAware.groovy @@ -12,10 +12,10 @@ trait ElasticSearchConfigAware { abstract GrailsApplication getGrailsApplication() ConfigObject getEsConfig() { - grailsApplication.config.elasticSearch as ConfigObject + grailsApplication?.config?.elasticSearch as ConfigObject } ConfigObject getMigrationConfig() { - (grailsApplication.config.elasticSearch as ConfigObject).migration as ConfigObject + (grailsApplication?.config?.elasticSearch as ConfigObject)?.migration as ConfigObject } } \ No newline at end of file From ca222bb0b1d09d48e6908ea4d44513c294d78a11 Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Fri, 3 Jun 2016 12:51:07 +0200 Subject: [PATCH 15/16] Prevent creating unnecessary indices --- .../mapping/SearchableClassMappingConfigurator.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy index 571c4ab0..c3dda9c5 100644 --- a/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy +++ b/src/main/groovy/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.groovy @@ -103,7 +103,7 @@ class SearchableClassMappingConfigurator implements ElasticSearchConfigAware { MappingMigrationStrategy migrationStrategy = migrationConfig?.strategy ? MappingMigrationStrategy.valueOf(migrationConfig?.strategy as String) : none def mappingConflicts = [] - Set indices = mappings.collect { it.indexName } as Set + Set indices = mappings.findAll { it.isRoot() }.collect { it.indexName } as Set //Install the mappings for each index all together indices.each { String indexName -> From a8d55c7eb73e10c00d96be844f8d09fca389a99b Mon Sep 17 00:00:00 2001 From: marcoscarceles Date: Fri, 3 Jun 2016 13:47:50 +0200 Subject: [PATCH 16/16] Updated docs and version (to 1.2.0) --- build.gradle | 2 +- gradle/publish.gradle | 4 ++-- src/docs/introduction/acknowledgements.adoc | 1 + src/docs/introduction/history.adoc | 9 +++---- src/docs/introduction/versioning.adoc | 26 ++++++++++++--------- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index c3c6634f..5d57587f 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { id "com.jfrog.bintray" version "1.2" } -version "2.3.0" +version "1.2.0" group "org.grails.plugins" apply plugin: 'maven-publish' diff --git a/gradle/publish.gradle b/gradle/publish.gradle index b57cafdd..88103929 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -15,7 +15,7 @@ if (bintrayProperties.exists()) { } title = "Elasticserach" desc = "An Elasticsearch plugin for Grails" - developers = [noam: 'Noam Y. Tenne', macrcos: 'Marcos Carceles', puneet: 'Puneet Behl', james: 'James Kleeh'] + developers = [noam: 'Noam Y. Tenne', marcos: 'Marcos Carceles', puneet: 'Puneet Behl', james: 'James Kleeh'] } } else { grailsPublish { @@ -25,7 +25,7 @@ if (bintrayProperties.exists()) { } title = "Elasticserach" desc = "An Elasticsearch plugin for Grails" - developers = [noam: 'Noam Y. Tenne', macrcos: 'Marcos Carceles', puneet: 'Puneet Behl', james: 'James Kleeh'] + developers = [noam: 'Noam Y. Tenne', marcos: 'Marcos Carceles', puneet: 'Puneet Behl', james: 'James Kleeh'] } } diff --git a/src/docs/introduction/acknowledgements.adoc b/src/docs/introduction/acknowledgements.adoc index 445180c2..8d436a50 100644 --- a/src/docs/introduction/acknowledgements.adoc +++ b/src/docs/introduction/acknowledgements.adoc @@ -8,6 +8,7 @@ Many thanks to all the users who reported issues and sent me pull requests. * https://github.com/stefanrother[Stefan Rother-Stübs (Dating Cafe)] * https://github.com/skies[Sven Kiesewetter (Dating Cafe)] * Michael Schwartz (Dating Cafe) +* https://github.com/marcoscarceles[Marcos Carceles] * https://github.com/puneetbehl[Puneet Behl] #### Authors and Contributors of the original plugin diff --git a/src/docs/introduction/history.adoc b/src/docs/introduction/history.adoc index 929603e4..7fe42575 100644 --- a/src/docs/introduction/history.adoc +++ b/src/docs/introduction/history.adoc @@ -2,11 +2,12 @@ ==== Grails 3.x version -* May 13, 2016 -** 2.3.0 -*** Upgraded to Elasticsearch 2.3.2, versioning now follow pattern described on https://github.com/noamt/elasticsearch-grails-plugin/issues/141[Issue 141] +* June 3, 2016 +** 1.2.0 +*** Upgraded to Elasticsearch 2.3.3 +**** Support for `elasticsearch-groovy` client removed (no available compatible version, pending on https://github.com/elastic/elasticsearch-groovy/issues/35[Issue 35]) +**** `delete` migration strategy has been replaced by `deleteIndex`, due to new Elasticsearch 2.x API's *** Removed test classes from plugin distribution -*** elasticsearch-groovy client removed (no available compatible version, pending on https://github.com/elastic/elasticsearch-groovy/issues/35[Issue 35]) * April 18, 2016 ** 1.0.0.1 diff --git a/src/docs/introduction/versioning.adoc b/src/docs/introduction/versioning.adoc index f8429bdd..77f4a859 100644 --- a/src/docs/introduction/versioning.adoc +++ b/src/docs/introduction/versioning.adoc @@ -6,20 +6,24 @@ The versioning model has changed. The version number of the plugin will reflect If necessary a 4th level point release number will be used for successive changes on the plugin's code with same version of Elasticsearch. ==== -`..`, where there isn't really a 1-to-1 plugin version to grails or es version, but we just increase our major or minor version by one, whenever there are breaking changes on either Grails or ES. Therefore have something that looks like: +`..`, where there isn't really a 1-to-1 plugin version to grails or es version, but we just increase our major or minor version by one, whenever there are breaking changes on either Grails or ES. +`..`, where there isn't really a 1-to-1 plugin version to grails or es version, but we just increase our major or minor version by one, whenever there are breaking changes on either Grails or ES. Therefore future release versions could look something similar to this (depending on Grails and Elasticsearch versions): |=== -| Plugin Version | Grails | Elasticsearch -| 0.0.4 | 2.4.x | 1.x -| 0.1.x | 2.4.x | 2.1.x -| 0.2.x | 2.4.x | 2.2.x -| 0.3.x | 2.4.x | (hypothetical) 3.0 -| 1.0.x | 3.1.x | 1.x -| 1.1.x | 3.1.x | 2.1.x -| 1.2.x | 3.1.x | 2.2.x -| 2.2.x | (hypothetical )3.2.x | 2.2.x -| 3.2.x | (hypothetical ) 4.0.x | 2.2.x +s| Plugin Version s| Grails s| Elasticsearch +s| 1.2.0 s| 3.1.x s| 2.3.x +s| 1.0.0.2 s| 3.1.x s| 1.x +s| 0.1.0 s| 2.4.x s| 2.1.x +s| 0.0.4.5 s| 2.4.x s| 1.x + | 0.2.x | 2.4.x | 2.3.x + | 0.3.x | 2.4.x | (hypothetical) 2.4.x + | 0.4.x | 2.4.x | (hypothetical) 5.0.x + | 1.3.x | 3.1.x | (hypothetical) 2.4.x + | 1.4.x | 3.1.x | (hypothetical) 5.0.x + | 2.3.x | (hypothetical) 3.2.x | (hypothetical) 2.3.x + | 3.3.x | (hypothetical) 4.0.x | (hypothetical) 2.3.x |=== +_Existing versions in_ *bold* Current version is *{revnumber}* (for Grails 2.x the latest version is *0.1.0*) \ No newline at end of file