Merge pull request #168 from CSCfi/qa

Export endpoints

aspnetcore/openshift/indexer/rahti2/template-indexer-cronjob-devel.yml

@@ -30,7 +30,7 @@ objects:
       failedJobsHistoryLimit: 2
       jobTemplate:
         spec:
-          activeDeadlineSeconds: 2400 # Can run for 40 minutes
+          activeDeadlineSeconds: 5400 # Can run for 90 minutes
           template:
             spec:
               restartPolicy: Never

aspnetcore/openshift/indexer/rahti2/template-indexer-cronjob-production.yml

@@ -30,7 +30,7 @@ objects:
       failedJobsHistoryLimit: 2
       jobTemplate:
         spec:
-          activeDeadlineSeconds: 2400 # Can run for 40 minutes
+          activeDeadlineSeconds: 5400 # Can run for 90 minutes
           template:
             spec:
               restartPolicy: Never

aspnetcore/openshift/indexer/rahti2/template-indexer-cronjob-qa.yml

@@ -30,7 +30,7 @@ objects:
       failedJobsHistoryLimit: 2
       jobTemplate:
         spec:
-          activeDeadlineSeconds: 2400 # Can run for 40 minutes
+          activeDeadlineSeconds: 5400 # Can run for 90 minutes
           template:
             spec:
               restartPolicy: Never

aspnetcore/src/ApiModels/Query/GetFundingCallQueryParameters.cs

@@ -6,7 +6,7 @@ namespace ResearchFi.Query;
 /// Query parameters for searching funding calls.
 /// </summary>
 /// <see cref="FundingCall"/>
-public class GetFundingCallQueryParameters : PaginationQueryParameters
+public class GetFundingCallQueryParameters
 {
     /// <summary>
     /// One of the fields nameFi, nameSV, nameEn contains the full text.

aspnetcore/src/ApiModels/Query/GetFundingDecisionQueryParameters.cs

@@ -6,7 +6,7 @@ namespace ResearchFi.Query;
 /// Query parameters for searching funding decisions.
 /// </summary>
 /// <see cref="FundingDecision"/>
-public class GetFundingDecisionQueryParameters : PaginationQueryParameters
+public class GetFundingDecisionQueryParameters
 {
     /// <summary>
     /// One of the fields nameFi, nameSV, nameEn contains the full text.

aspnetcore/src/ApiModels/Query/GetInfrastructuresQueryParameters.cs

@@ -3,7 +3,7 @@
 /// <summary>
 /// Hakuparametrit infrastruktuurien hakemiseen.
 /// </summary>
-public class GetInfrastructuresQueryParameters : PaginationQueryParameters
+public class GetInfrastructuresQueryParameters
 {
     /// <summary>
     /// Infrastruktuurin nimi.

aspnetcore/src/ApiModels/Query/GetPublicationsQueryParameters.cs

@@ -6,7 +6,7 @@ namespace ResearchFi.Query;
 /// Query parameters for searching publications.
 /// </summary>
 /// <see cref="Publication"/>
-public class GetPublicationsQueryParameters : PaginationQueryParameters
+public class GetPublicationsQueryParameters
 {
     /// <summary>
     /// The field name contains text.

aspnetcore/src/ApiModels/Query/PaginationQueryParameters.cs

@@ -19,7 +19,7 @@ public int PageNumber
     }
 
     /// <summary>
-    /// Number of results on page. Optional. Default value 20. Maximum permissible value 100. Maximum possible result set of 10,000 results.
+    /// Number of results on page. Optional. Default value 20. Maximum permissible value 100. Maximum possible result set of 10000 results.
     /// </summary>
     public int PageSize
     {

aspnetcore/src/ApiModels/Query/SearchAfterQueryParameters.cs

@@ -0,0 +1,30 @@
+namespace ResearchFi.Query;
+
+/// <summary>
+/// Vientiin liittyvät tiedot.
+/// </summary>
+public class SearchAfterQueryParameters
+{
+    private const int DefaultPageSize = 50;
+    private const int MaximumPageSize = 1000;
+    private int _pageSize = DefaultPageSize;
+    private long? _nextPageToken = null;
+
+    /// <summary>
+    /// Number of results on page. Optional. Default value 50. Maximum permissible value 1000.
+    /// </summary>
+    public int PageSize
+    {
+        get => _pageSize;
+        set => _pageSize = value < 1 ? DefaultPageSize : (value > MaximumPageSize ? MaximumPageSize : value);
+    }
+
+    /// <summary>
+    /// Value from previous query response header "x-next-page-token". Leave empty in the first query.
+    /// </summary>
+    public long? NextPageToken
+    {
+        get => _nextPageToken;
+        set => _nextPageToken = value;
+    }
+}

aspnetcore/src/ElasticService/ElasticSearchIndexService.cs

@@ -32,7 +32,7 @@ public async Task IndexAsync(string indexName, List<object> entities, Type model
         await IndexEntities(indexToCreate, entities, modelType);
 
         // Switch indexes
-        await SwitchIndexes(indexName, indexToCreate, indexToDelete);
+        await SwitchIndexes(indexName, indexToCreate, indexToDelete, modelType.Name);
 
         _logger.LogDebug("{EntityType:l}: Indexing to {IndexName:l} complete", modelType.Name, indexName);
     }
@@ -43,8 +43,9 @@ public async Task IndexChunkAsync(string indexToCreate, List<object> entities, T
         await IndexEntities(indexToCreate, entities, modelType);
     }
 
-    public async Task SwitchIndexes(string indexName, string indexToCreate, string indexToDelete)
+    public async Task SwitchIndexes(string indexName, string indexToCreate, string indexToDelete, string modelTypeName)
     {
+        _logger.LogInformation($"{modelTypeName}: Switch indexes start: indexName={indexName}, indexToCreate={indexToCreate}, indexToDelete={indexToDelete}");
         // Wait for new index to be operational.
         await _elasticClient.Cluster
             .HealthAsync(selector: s => s
@@ -61,6 +62,7 @@ await _elasticClient.Indices.BulkAliasAsync(r => r
 
         // Delete the old index if it exists.
         await _elasticClient.Indices.DeleteAsync(indexToDelete, d => d.RequestConfiguration(x => x.AllowedStatusCodes(404)));
+        _logger.LogInformation($"{modelTypeName}: Switch indexes complete");
     }
 
     public async Task<(string indexToCreate, string indexToDelete)> GetIndexNames(string indexName)

aspnetcore/src/ElasticService/ElasticSearchQueryGenerators/IQueryGenerator.cs

@@ -5,5 +5,6 @@ namespace CSC.PublicApi.ElasticService.ElasticSearchQueryGenerators;
 public interface IQueryGenerator<TIn, TOut> where TOut : class
 {
     Func<SearchDescriptor<TOut>, ISearchRequest> GenerateQuery(TIn searchParameters, int pageNumber, int pageSize);
+    Func<SearchDescriptor<TOut>, ISearchRequest> GenerateQuerySearchAfter(TIn searchParameters, int pageSize, long? searchAfter);
     Func<SearchDescriptor<TOut>, ISearchRequest> GenerateSingleQuery(string id);
 }

aspnetcore/src/ElasticService/ElasticSearchQueryGenerators/QueryGeneratorBase.cs

@@ -22,6 +22,27 @@ public Func<SearchDescriptor<TOut>, ISearchRequest> GenerateQuery(TIn searchPara
             .Query(GenerateQueryForSearch(searchParameters));
     }
 
+    public Func<SearchDescriptor<TOut>, ISearchRequest> GenerateQuerySearchAfter(TIn searchParameters, int pageSize, long? searchAfter)
+    {
+        var indexName = _configuration.GetIndexNameForType(typeof(TOut));
+
+        if (searchAfter == null) {
+            return descriptor => descriptor
+                .Index(indexName)
+                .Take(pageSize)
+                .Sort(sort => sort.Ascending(SortSpecialField.DocumentIndexOrder))
+                .Query(GenerateQueryForSearch(searchParameters));
+        }
+        else {
+            return descriptor => descriptor
+                .Index(indexName)
+                .Take(pageSize)
+                .Sort(sort => sort.Ascending(SortSpecialField.DocumentIndexOrder))
+                .Query(GenerateQueryForSearch(searchParameters))
+                .SearchAfter(searchAfter);
+        }
+    }
+
     public Func<SearchDescriptor<TOut>, ISearchRequest> GenerateSingleQuery(string id)
     {
         var indexName = _configuration.GetIndexNameForType(typeof(TOut));

aspnetcore/src/ElasticService/ElasticSearchService.cs

@@ -32,6 +32,27 @@ public ElasticSearchService(
         return (searchResult.Documents, new SearchResult(pageNumber, pageSize, searchResult.HitsMetadata?.Total.Value));
     }
 
+    public async Task<(IEnumerable<TOut>, SearchAfterResult)> SearchAfter(TIn parameters, int pageSize, long? searchAfter)
+    {
+        var query = _queryGenerator.GenerateQuerySearchAfter(parameters, pageSize, searchAfter);
+
+        var searchResult = await _elasticClient.SearchAsync(query);
+
+        if (Debugger.IsAttached)
+        {
+            // Enables seeing the query sent to elastic and the response in the log when debugging.
+            Console.WriteLine(searchResult.DebugInformation);            
+        }
+
+        long? searchAfterValue = null;
+
+        if (searchResult.Hits.Count > 0) {
+            searchAfterValue = (long)searchResult.Hits.Last().Sorts.First();
+        }
+
+        return (searchResult.Documents, new SearchAfterResult(searchAfterValue, pageSize));
+    }
+
     public async Task<TOut?> GetSingle(string id)
     {
         var query = _queryGenerator.GenerateSingleQuery(id);

aspnetcore/src/ElasticService/IElasticSearchIndexService.cs

@@ -48,6 +48,7 @@ public interface IElasticSearchIndexService
     /// <param name="indexName"></param>
     /// <param name="indexToCreate"></param>
     /// <param name="indexToDelete"></param>
+    /// <param name="modelTypeName"></param>
     /// <returns></returns>
-    Task SwitchIndexes(string indexName, string indexToCreate, string indexToDelete);
+    Task SwitchIndexes(string indexName, string indexToCreate, string indexToDelete, string modelTypeName);
 }

aspnetcore/src/ElasticService/ISearchService.cs

@@ -3,6 +3,6 @@
 public interface ISearchService<TIn, TOut> where TOut : class
 {
     Task<(IEnumerable<TOut>, SearchResult)> Search(TIn searchParameters, int pageNumber, int pageSize);
-
+    Task<(IEnumerable<TOut>, SearchAfterResult)> SearchAfter(TIn searchParameters, int pageSize, long? searchAfter);
     Task<TOut?> GetSingle(string id);
 }

aspnetcore/src/ElasticService/SearchAfterResult.cs

@@ -0,0 +1,12 @@
+namespace CSC.PublicApi.ElasticService;
+
+public class SearchAfterResult
+{
+    public long? SearchAfter { get; }
+    public int PageSize { get; }
+    public SearchAfterResult(long? searchAfter, int pageSize)
+    {
+        SearchAfter = searchAfter;
+        PageSize = pageSize;
+    }
+}

aspnetcore/src/Indexer/Indexer.cs

@@ -177,7 +177,7 @@ private async Task IndexEntities(string indexName, IIndexRepository repository,
                 } while (numOfResults >= takeAmount - 1);
 
                 // Activate new index and delete old
-                await _indexService.SwitchIndexes(indexName, indexToCreate, indexToDelete);
+                await _indexService.SwitchIndexes(indexName, indexToCreate, indexToDelete, type.Name);
                 _logger.LogInformation("{EntityType:l}: Recreated index {IndexName:l}, {ElasticsearchDocumentCount} documents", type.Name, indexName, processedCount);
             }
             else

aspnetcore/src/Interface/Configuration/SwaggerExtensions.cs

@@ -76,6 +76,7 @@ public static void UseSwaggerAndSwaggerUI(this WebApplication app)
             foreach (var description in app.Services.GetRequiredService<IApiVersionDescriptionProvider>().ApiVersionDescriptions)
             {
                 options.SwaggerEndpoint($"{description.GroupName}/swagger.json", description.GroupName.ToUpperInvariant());
+                options.ConfigObject.AdditionalItems.Add("syntaxHighlight", false); // disable to improve performance with large responses
             }
         });
     }

aspnetcore/src/Interface/Controllers/FundingCallController.cs

@@ -30,7 +30,7 @@ public FundingCallController(
     /// <summary>
     /// Endpoint for filtering funding calls using the specified query parameters.
     /// </summary>
-    /// <param name="queryParameters">The query parameters for filtering the results.</param>
+    /// <param name="fundingCallQueryParameters">The query parameters for filtering the results.</param>
     /// <returns>Paged search result as a collection of <see cref="FundingCall"/> objects.</returns>
     /// <response code="200">Ok.</response>
     /// <response code="401">Unauthorized.</response>
@@ -43,9 +43,9 @@ public FundingCallController(
     [ProducesResponseType(typeof(IEnumerable<FundingCall>), StatusCodes.Status200OK)]
     [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
     [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
-    public async Task<IEnumerable<FundingCall>> Get([FromQuery] GetFundingCallQueryParameters queryParameters)
+    public async Task<IEnumerable<FundingCall>> Get([FromQuery] GetFundingCallQueryParameters fundingCallQueryParameters, [FromQuery] PaginationQueryParameters paginationQueryParameters)
     {
-        var (fundingCalls, searchResult) = await _service.GetFundingCalls(queryParameters);
+        var (fundingCalls, searchResult) = await _service.GetFundingCalls(fundingCallQueryParameters, paginationQueryParameters);
 
         ResponseHelper.AddPaginationResponseHeaders(HttpContext, searchResult);
 

aspnetcore/src/Interface/Controllers/FundingCallExportController.cs

@@ -0,0 +1,54 @@
+using CSC.PublicApi.Interface.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using ResearchFi.FundingCall;
+using ResearchFi.Query;
+using Serilog;
+
+namespace CSC.PublicApi.Interface.Controllers;
+
+[ApiController]
+[ApiVersion(ApiConstants.ApiVersion1)]
+[Route("v{version:apiVersion}/funding-calls-export")]
+public class FundingCallExportController : ControllerBase
+{
+    private readonly ILogger<FundingCallExportController> _logger;
+    private readonly IFundingCallService _service;
+    private readonly IDiagnosticContext _diagnosticContext;
+
+    public FundingCallExportController(
+        ILogger<FundingCallExportController> logger,
+        IFundingCallService service,
+        IDiagnosticContext diagnosticContext)
+    {
+        _logger = logger;
+        _service = service;
+        _diagnosticContext = diagnosticContext;
+        _diagnosticContext.Set(ApiConstants.LogResourceType_PropertyName, ApiConstants.LogResourceType_FundingCall);
+    }
+
+    /// <summary>
+    /// Endpoint for bypassing the limit of 10000 records for funding calls.
+    /// </summary>
+    /// <param name="fundingCallQueryParameters">The query parameters for filtering the results.</param>
+    /// <returns>Paged search result as a collection of <see cref="FundingCall"/> objects.</returns>
+    /// <response code="200">Ok.</response>
+    /// <response code="401">Unauthorized.</response>
+    /// <response code="403">Forbidden.</response>
+    [HttpGet(Name = "GetFundingCallExport")]
+    [MapToApiVersion(ApiConstants.ApiVersion1)]
+    [Authorize(Policy = ApiPolicies.FundingCall.Read)]
+    [Produces(ApiConstants.ContentTypeJson)]
+    [Consumes(ApiConstants.ContentTypeJson)]
+    [ProducesResponseType(typeof(IEnumerable<FundingCall>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
+    public async Task<IEnumerable<FundingCall>> Get([FromQuery] GetFundingCallQueryParameters fundingCallQueryParameters, [FromQuery] SearchAfterQueryParameters searchAfterQueryParameters)
+    {
+        var (fundingCalls, searchAfterResult) = await _service.GetFundingCallsSearchAfter(fundingCallQueryParameters, searchAfterQueryParameters);
+
+        ResponseHelper.AddPaginationResponseHeadersSearchAfter(HttpContext, searchAfterResult);
+
+        return fundingCalls;
+    }
+}

aspnetcore/src/Interface/Controllers/FundingDecisionController.cs

@@ -32,6 +32,7 @@ public FundingDecisionController(
     /// <summary>
     /// Endpoint for filtering funding decisions using the specified query parameters.
     /// </summary>
+    /// <param name="fundingDecisionQueryParameters">The query parameters for filtering the results.</param>
     /// <returns>Paged search result as a collection of <see cref="FundingDecision"/> objects.</returns>
     /// <response code="200">Ok.</response>
     /// <response code="401">Unauthorized.</response>
@@ -44,9 +45,9 @@ public FundingDecisionController(
     [ProducesResponseType(typeof(IEnumerable<FundingDecision>), StatusCodes.Status200OK)]
     [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
     [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
-    public async Task<IEnumerable<FundingDecision>> Get([FromQuery] GetFundingDecisionQueryParameters queryParameters)
+    public async Task<IEnumerable<FundingDecision>> Get([FromQuery] GetFundingDecisionQueryParameters fundingDecisionQueryParameters, [FromQuery] PaginationQueryParameters paginationQueryParameters)
     {
-        var (fundingDecisions, searchResult) = await _service.GetFundingDecisions(queryParameters);
+        var (fundingDecisions, searchResult) = await _service.GetFundingDecisions(fundingDecisionQueryParameters, paginationQueryParameters);
 
         ResponseHelper.AddPaginationResponseHeaders(HttpContext, searchResult);