umbrella project queries can extract shared parameters

lib/controllers/v1/observations_controller.js

@@ -1053,7 +1053,9 @@ ObservationsController.qualityGrades = async req => {
 
 ObservationsController.umbrellaSubprojectsAggregation = async projects => {
   const aggs = { umbrellaSubprojects: { filters: { filters: { } } } };
-  const queryFilters = await ObservationQueryBuilder.projectsQueryFilters( projects );
+  const queryFilters = await ObservationQueryBuilder.projectsQueryFilters(
+    projects, { splitByProject: true }
+  );
   _.each( queryFilters, q => {
     aggs.umbrellaSubprojects.filters.filters[`project_${q.project.id}`] = {
       bool: {

lib/models/observation_query_builder.js

@@ -1031,7 +1031,10 @@ ObservationQueryBuilder.reqToElasticQueryComponents = async req => {
 
   // Set the place filters, which gets REALLY complicated when trying to decide
   // when to search on private places or not
-  if ( params.place_id && params.place_id !== "any" ) {
+  if ( params.place_id
+    && params.place_id !== "any"
+    && !( _.isArray( params.place_id ) && _.isEmpty( params.place_id ) )
+  ) {
     // This is the basic filter of places everyone should see
     const publicPlaceFilter = esClient.termFilter( "place_ids.keyword", params.place_id );
     if ( req.userSession ) {
@@ -1044,7 +1047,9 @@ ObservationQueryBuilder.reqToElasticQueryComponents = async req => {
     }
   }
 
-  if ( params.not_in_place ) {
+  if ( params.not_in_place
+    && !( _.isArray( params.not_in_place ) && _.isEmpty( params.not_in_place ) )
+  ) {
     inverseFilters.push( esClient.termFilter( "place_ids.keyword", params.not_in_place ) );
   }
 
@@ -1663,6 +1668,80 @@ ObservationQueryBuilder.applyCollectionProjectRules = async ( req, options = { }
   }
 };
 
+ObservationQueryBuilder.minimizedUmbrellaQueryFilters = async (
+  combinedParameters, collectionsQueries, options = { }
+) => {
+  let queryFilters;
+  const commonParameters = _.mapValues(
+    _.omit( combinedParameters, ["place_id", "not_in_place"] ),
+    values => _.first( _.keys( values ) )
+  );
+  // if there are any not_in_place filters, a more complicated query is
+  // required using conditional `should` clauses for those projects
+  if ( combinedParameters.not_in_place ) {
+    const completePlaceIDs = _.compact( _.map( _.reject(
+      collectionsQueries,
+      collectionsQuery => _.has( collectionsQuery, "not_in_place" )
+    ), "place_id" ) );
+    const {
+      search_filters: completePlacefilters
+    } = await ObservationQueryBuilder.reqToElasticQueryComponents( {
+      query: {
+        place_id: _.union(
+          ..._.map( completePlaceIDs, util.paramArray )
+        )
+      }
+    } );
+    const queriesWithNotInPlace = _.filter(
+      collectionsQueries,
+      collectionsQuery => _.has( collectionsQuery, "not_in_place" )
+    );
+    const allCollectionPlaceRules = await Promise.all(
+      _.map( queriesWithNotInPlace, collectionsQuery => (
+        ObservationQueryBuilder.reqToElasticQueryComponents( {
+          query: {
+            place_id: collectionsQuery.place_id,
+            not_in_place: collectionsQuery.not_in_place
+          }
+        } )
+      ) )
+    );
+    const shoulds = completePlacefilters;
+    _.each( allCollectionPlaceRules, rules => {
+      shoulds.push( {
+        bool: {
+          filter: rules.search_filters,
+          must_not: rules.inverse_filters
+        }
+      } );
+    } );
+    queryFilters = await ObservationQueryBuilder.reqToElasticQueryComponents( {
+      query: {
+        ...commonParameters,
+        filters: [{
+          bool: {
+            should: shoulds
+          }
+        }]
+      }
+    } );
+  } else {
+    // no projects use not_in_place filters, so search on a union of every
+    // place_id from every project
+    commonParameters.place_id = _.union(
+      ..._.map( _.keys( combinedParameters.place_id ), util.paramArray )
+    );
+    queryFilters = await ObservationQueryBuilder.reqToElasticQueryComponents( {
+      query: commonParameters,
+      userSession: options.userSession
+    } );
+  }
+  return [{
+    filters: queryFilters.search_filters,
+    inverse_filters: queryFilters.inverse_filters
+  }];
+};
+
 ObservationQueryBuilder.projectsQueryFilters = async ( projects, options = {} ) => {
   if ( options.userSession ) {
     // the Promise.all below will ultimately call userSession.getBlocks for each
@@ -1671,6 +1750,33 @@ ObservationQueryBuilder.projectsQueryFilters = async ( projects, options = {} )
     // each project without needing a separate query
     await options.userSession.getBlocks( );
   }
+  const collectionsQueries = _.compact(
+    _.map( projects, ObservationQueryBuilder.collectionProjectQuery )
+  );
+  const combinedParameters = { };
+  _.each( collectionsQueries, collectionQuery => {
+    _.each( collectionQuery, ( value, parameter ) => {
+      combinedParameters[parameter] ||= { };
+      combinedParameters[parameter][value] ||= 0;
+      combinedParameters[parameter][value] += 1;
+    } );
+  } );
+  const nonPlaceParametersConsistent = _.every(
+    _.omit( combinedParameters, ["place_id", "not_in_place"] ),
+    ( values, parameter ) => (
+      _.size( values ) === 1
+      && combinedParameters[parameter][_.keys( values )[0]] === projects.length
+    )
+  );
+  // if there are multiple projects, and all filters but place filters are
+  // exactly the same, the query can be made mode efficient by group those
+  // parameters and handling place_id filters separately
+  if ( projects.length > 1 && nonPlaceParametersConsistent && !options.splitByProject ) {
+    return ObservationQueryBuilder.minimizedUmbrellaQueryFilters(
+      combinedParameters, collectionsQueries, options
+    );
+  }
+
   const queryFilters = await Promise.all(
     _.map( projects, p => ObservationQueryBuilder.projectRulesQueryFilters( p, options ) )
   );
@@ -1681,15 +1787,15 @@ ObservationQueryBuilder.projectsQueryFilters = async ( projects, options = {} )
   } ) );
 };
 
-ObservationQueryBuilder.projectRulesQueryFilters = async ( collection, options = {} ) => {
-  let collectParamsHash = {};
+ObservationQueryBuilder.collectionProjectQuery = collection => {
   if ( collection.project_type === "umbrella" ) {
-    return { searchFilters: [{ term: { id: -1 } }] };
+    return null;
   }
   if ( collection.project_type !== "collection" ) {
-    return { searchFilters: [{ term: { "project_ids.keyword": collection.id } }] };
+    return { project_id: collection.id };
   }
   // make an object of all the obs search parameters for this project
+  const collectParamsHash = {};
   let membersOnly;
   _.each( collection.search_parameters, p => {
     // make sure all values are strings, as they would be in HTTP GET params
@@ -1709,26 +1815,35 @@ ObservationQueryBuilder.projectRulesQueryFilters = async ( collection, options =
     && !collectParamsHash.month
     && !collectParamsHash.observed_on
     && _.isEmpty( collection.project_observation_rules )
-    && !membersOnly ) {
+    && !membersOnly
+  ) {
     // The new-style project does not have any major search parameters.
     // Return an unmatchable filter indicating no obervations in this project
-    collectParamsHash = { id: "-1" };
+    return null;
   }
   if ( membersOnly ) {
     if ( collectParamsHash.user_id ) {
       collectParamsHash.user_id = _.intersection(
         collection.user_ids, _.map( collectParamsHash.user_id.split( "," ), Number )
       ).join( "," );
-      if ( _.isEmpty( collectParamsHash.user_id ) ) {
-        collectParamsHash.user_id = "-1";
-      }
     } else {
       collectParamsHash.user_id = collection.user_ids.join( "," );
     }
+    if ( _.isEmpty( collectParamsHash.user_id ) ) {
+      return null;
+    }
   }
   if ( _.isEmpty( collectParamsHash ) ) {
     // The new-style project does not have search parameters.
     // Return an unmatchable filter indicating no obervations in this project
+    return null;
+  }
+  return collectParamsHash;
+};
+
+ObservationQueryBuilder.projectRulesQueryFilters = async ( collection, options = {} ) => {
+  const collectParamsHash = ObservationQueryBuilder.collectionProjectQuery( collection );
+  if ( _.isEmpty( collectParamsHash ) ) {
     return { searchFilters: [{ term: { id: -1 } }] };
   }
   // turn the HTTP-like params into ES query filters and return