Release 1.25 (#191)

* DimPublication model update (#181) Update DimPublication properties InternationalCollaboration, BusinessCollaboration and SelfArchivedCode * CSCTTV-3248 add field url in profile only tables (#182) * CSCTTV-3248 dim_web_link model change Add foreign keys to DimProfileOnlyDataset, DimProfileOnlyFundingDecision and DimProfileOnlyResearchActivity * CSCTTV-3248 Add Url handling in ORCID import. Add Url handling in profile editor and Elasticsearch models. * CSCTTV-3315 Add cooperation choices in Elasticsearch person index (#183) * CSCTTV-3275 Admin command to update all profiles in Elasticsearch (#184) * CSCTTV-3288 Admin command to update ORCID data for all profiles (#185) * Reduce delay time to 1 second between admin API calls (#186) * Add application health check endpoint (#187) * Edit application health check (#188) Simplify application health check. Database connection check removed and controller now responds immediately with status code 200. * Update DimPublication model to be compatible with QA and production database model. (#189) * CSCTTV-3275 and CSCTTV-3288 increase task queu capacity (#190)
CSCfi · Jun 21, 2023 · 50caa5e · 50caa5e
1 parent 094f5e6
commit 50caa5e
Show file tree

Hide file tree

Showing 28 changed files with 578 additions and 104 deletions.
diff --git a/aspnetcore/src/api.Tests/Services_Tests/TtvSqlServiceTest.cs b/aspnetcore/src/api.Tests/Services_Tests/TtvSqlServiceTest.cs
@@ -353,9 +353,9 @@ public void Test_getSqlQuery_Select_DimWebLink_01()
             int dimKnownPersonId = 9997;
             List<int> existingIds = new() {};
             string expectedSqlString =
-                    @"SELECT id as 'Id', dim_registered_data_source_id AS 'DimRegisteredDataSourceId'
+                    @$"SELECT id as 'Id', dim_registered_data_source_id AS 'DimRegisteredDataSourceId'
                         FROM dim_web_link
-                        WHERE dim_known_person_id=9997 AND id!=-1 AND dim_registered_data_source_id!=-1 AND dim_registered_data_source_id IS NOT NULL";
+                        WHERE dim_known_person_id=9997 AND id!=-1 AND dim_registered_data_source_id!=-1 AND dim_registered_data_source_id IS NOT NULL AND source_id!='{Constants.SourceIdentifiers.PROFILE_API}'";
             // Act
             string actualSqlString = ttvSqlService.GetSqlQuery_Select_DimWebLink(dimKnownPersonId, existingIds);
             // Assert
@@ -370,9 +370,9 @@ public void Test_getSqlQuery_Select_DimWebLink_02()
             int dimKnownPersonId = 9997;
             List<int> existingIds = new() { 333, 444, 555 };
             string expectedSqlString =
-                    @"SELECT id as 'Id', dim_registered_data_source_id AS 'DimRegisteredDataSourceId'
+                    @$"SELECT id as 'Id', dim_registered_data_source_id AS 'DimRegisteredDataSourceId'
                         FROM dim_web_link
-                        WHERE dim_known_person_id=9997 AND id!=-1 AND dim_registered_data_source_id!=-1 AND dim_registered_data_source_id IS NOT NULL AND id NOT IN (333,444,555)";
+                        WHERE dim_known_person_id=9997 AND id!=-1 AND dim_registered_data_source_id!=-1 AND dim_registered_data_source_id IS NOT NULL AND id NOT IN (333,444,555) AND source_id!='{Constants.SourceIdentifiers.PROFILE_API}'";
             // Act
             string actualSqlString = ttvSqlService.GetSqlQuery_Select_DimWebLink(dimKnownPersonId, existingIds);
             // Assert

diff --git a/aspnetcore/src/api/Background/BackgroundProfileData.cs b/aspnetcore/src/api/Background/BackgroundProfileData.cs
@@ -5,6 +5,9 @@
 using Microsoft.Extensions.DependencyInjection;
 using api.Models.ProfileEditor.Items;
 using AutoMapper;
+using Microsoft.EntityFrameworkCore;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace api.Services
 {
@@ -34,6 +37,7 @@ public async Task<ElasticsearchPerson> GetProfiledataForElasticsearch(string orc
             using IServiceScope scope = _serviceScopeFactory.CreateScope();
             IUserProfileService localUserProfileService = scope.ServiceProvider.GetRequiredService<IUserProfileService>();
             IMapper mapper = scope.ServiceProvider.GetRequiredService<IMapper>();
+            TtvContext localTtvContext = scope.ServiceProvider.GetRequiredService<TtvContext>();
 
             // Get profile data
             ProfileEditorDataResponse profileEditorDataResponse =
@@ -47,6 +51,25 @@ await localUserProfileService.GetProfileDataAsync(
             ElasticsearchPerson elasticsearchPerson = mapper.Map<ElasticsearchPerson>(profileEditorDataResponse);
             elasticsearchPerson.id = orcidId;
 
+            // Add cooperation choices
+            List<DimUserChoice> userChoices =
+                await localTtvContext.DimUserChoices.TagWith("Get user choices for Elasticsearch")
+                .Where(duc => duc.DimUserProfileId == userprofileId && duc.UserChoiceValue == true)
+                .Include(duc => duc.DimReferencedataIdAsUserChoiceLabelNavigation).AsNoTracking().ToListAsync();
+
+            foreach (DimUserChoice uc in userChoices)
+            {
+                elasticsearchPerson.cooperation.Add(
+                    new ElasticsearchCooperation()
+                    {
+                        Id = uc.DimReferencedataIdAsUserChoiceLabelNavigation.Id,
+                        NameFi = uc.DimReferencedataIdAsUserChoiceLabelNavigation.NameFi,
+                        NameEn = uc.DimReferencedataIdAsUserChoiceLabelNavigation.NameEn,
+                        NameSv = uc.DimReferencedataIdAsUserChoiceLabelNavigation.NameSv
+                    }
+                );
+            }
+
             return elasticsearchPerson;
         }
     }

diff --git a/aspnetcore/src/api/Background/BackgroundTaskQueue.cs b/aspnetcore/src/api/Background/BackgroundTaskQueue.cs
@@ -27,9 +27,7 @@ public BackgroundTaskQueue()
         // BoundedChannelFullMode.Wait will cause calls to WriteAsync() to return a task,
         // which completes only when space became available. This leads to backpressure,
         // in case too many publishers/calls start accumulating.
-
-        // TODO: Hard coded capacity for now. Add to configuration?
-        var options = new BoundedChannelOptions(capacity: 100)
+        var options = new BoundedChannelOptions(capacity: 1000)
         {
             FullMode = BoundedChannelFullMode.Wait
         };

diff --git a/aspnetcore/src/api/Controllers/AdminController.cs b/aspnetcore/src/api/Controllers/AdminController.cs
@@ -9,6 +9,7 @@
 using Microsoft.AspNetCore.Mvc;
 using api.Models.Ttv;
 using api.Models.Log;
+using System.Threading;
 
 namespace api.Controllers
 {
@@ -333,7 +334,7 @@ public async Task<IActionResult> UpdateAllUserprofilesInElasticsearch()
             }
 
             LogUserIdentification logUserIdentification = this.GetLogUserIdentification();
-            await _adminService.UpdateAllUserprofilesInElasticsearch(logUserIdentification);
+            await _adminService.UpdateAllUserprofilesInElasticsearch(logUserIdentification, Request.Scheme, Request.Host);
             return Ok();
         }
 
@@ -365,5 +366,41 @@ public async Task<IActionResult> AddNewTtvDataInUserProfile(int dimUserProfileId
 
             return Ok();
         }
+
+
+        /// <summary>
+        /// Admin: Update ORCID data for all user profiles
+        /// </summary>
+        [HttpPost]
+        [Route("/[controller]/userprofile/orcidupdate/all")]
+        public async Task<IActionResult> UpdateOrcidDataForAllUserprofiles()
+        {
+            // Validate request data
+            if (!ModelState.IsValid)
+            {
+                return BadRequest();
+            }
+
+            // Check admin token authorization
+            if (!IsAdminTokenAuthorized(Configuration))
+            {
+                return Unauthorized();
+            }
+
+            LogUserIdentification logUserIdentification = this.GetLogUserIdentification();
+            await _adminService.UpdateOrcidDataForAllUserprofiles(logUserIdentification, Request.Scheme, Request.Host);
+            return Ok();
+        }
+
+
+        /// <summary>
+        /// Admin: Enpoint for checking application health
+        /// </summary>
+        [HttpGet]
+        [Route("/[controller]/health")]
+        public ActionResult Health(CancellationToken cancellationToken)
+        { 
+            return Ok();
+        }
     }
 }
diff --git a/aspnetcore/src/api/Models/Common/Constants.cs b/aspnetcore/src/api/Models/Common/Constants.cs
@@ -85,6 +85,12 @@ public static class Cache
             public const int MEMORY_CACHE_EXPIRATION_SECONDS_LONG = 3600;
         }
 
+        public static class Delays
+        {
+            public const int ADMIN_UPDATE_ALL_PROFILES_IN_ELASTICSEARCH_DELAY_BETWEEN_API_CALLS_MS = 500;
+            public const int ADMIN_UPDATE_ALL_PROFILES_ORCID_DATA_DELAY_BETWEEN_API_CALLS_MS = 500;
+        }
+
         public static class OrganizationIds
         {
             public const string OKM = "02w52zt87";

diff --git a/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchActivityAndReward.cs b/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchActivityAndReward.cs
@@ -33,6 +33,7 @@ public ElasticsearchActivityAndReward()
             DepartmentNameFi = "";
             DepartmentNameSv = "";
             DepartmentNameEn = "";
+            Url = "";
             sector = new List<ElasticsearchSector> { };
         }
 
@@ -59,6 +60,7 @@ public ElasticsearchActivityAndReward()
         public string DepartmentNameFi { get; set; }
         public string DepartmentNameSv { get; set; }
         public string DepartmentNameEn { get; set; }
+        public string Url { get; set; }
 
         [Nested]
         [PropertyName("sector")]

diff --git a/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchCooperation.cs b/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchCooperation.cs
@@ -0,0 +1,14 @@
+namespace api.Models.Elasticsearch
+{
+    public partial class ElasticsearchCooperation
+    {
+        public ElasticsearchCooperation()
+        {
+        }
+
+        public int Id { get; set; }
+        public string NameFi { get; set; }
+        public string NameEn { get; set; }
+        public string NameSv { get; set; }
+    }
+}
diff --git a/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchFundingDecision.cs b/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchFundingDecision.cs
@@ -26,6 +26,7 @@ public ElasticsearchFundingDecision()
             AmountInEur = -1;
             AmountInFundingDecisionCurrency = null;
             FundingDecisionCurrencyAbbreviation = null;
+            Url = "";
         }
 
         public int ProjectId { get; set; }
@@ -51,5 +52,6 @@ public ElasticsearchFundingDecision()
         public decimal AmountInEur { get; set; }
         public decimal? AmountInFundingDecisionCurrency { get; set; }
         public string? FundingDecisionCurrencyAbbreviation { get; set; }
+        public string Url { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchPerson.cs b/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchPerson.cs
@@ -10,6 +10,7 @@ public ElasticsearchPerson()
             id = "";
             personal = new();
             activity = new();
+            cooperation = new();
             uniqueDataSources = new();
         }
 
@@ -18,6 +19,7 @@ public ElasticsearchPerson(string orcidId)
             id = orcidId;
             personal = new();
             activity = new();
+            cooperation = new();
             uniqueDataSources = new();
         }
 
@@ -26,6 +28,7 @@ public ElasticsearchPerson(string orcidId)
         [Nested]
         [PropertyName("activity")]
         public ElasticsearchActivity activity { get; set; }
+        public List<ElasticsearchCooperation> cooperation { get; set; }
         public List<ElasticsearchSource> uniqueDataSources { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchResearchDataset.cs b/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchResearchDataset.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
 namespace api.Models.Elasticsearch
 {
@@ -14,6 +15,7 @@ public ElasticsearchResearchDataset()
             DescriptionFi = "";
             DescriptionSv = "";
             DescriptionEn = "";
+            Url = "";
             DatasetCreated = null;
             PreferredIdentifiers = new List<ElasticsearchPreferredIdentifier>();
         }
@@ -27,6 +29,7 @@ public ElasticsearchResearchDataset()
         public string DescriptionFi { get; set; }
         public string DescriptionSv { get; set; }
         public string DescriptionEn { get; set; }
+        public string Url { get; set; }
         public int? DatasetCreated { get; set; }
         public List<ElasticsearchPreferredIdentifier> PreferredIdentifiers { get; set; }
     }

diff --git a/aspnetcore/src/api/Models/ProfileEditor/Items/ProfileEditorActivityAndReward.cs b/aspnetcore/src/api/Models/ProfileEditor/Items/ProfileEditorActivityAndReward.cs
@@ -29,6 +29,7 @@ public ProfileEditorActivityAndReward()
             DepartmentNameFi = "";
             DepartmentNameSv = "";
             DepartmentNameEn = "";
+            Url = "";
             sector = new List<ProfileEditorSector> { };
         }
 
@@ -55,6 +56,7 @@ public ProfileEditorActivityAndReward()
         public string DepartmentNameFi { get; set; }
         public string DepartmentNameSv { get; set; }
         public string DepartmentNameEn { get; set; }
+        public string Url { get; set; }
 
         // Fields required in Elasticsearch person index. Elasticsearch model is mapped from ProfileEditor model.
         public List<ProfileEditorSector> sector { get; set; }

diff --git a/aspnetcore/src/api/Models/ProfileEditor/Items/ProfileEditorFundingDecision.cs b/aspnetcore/src/api/Models/ProfileEditor/Items/ProfileEditorFundingDecision.cs
@@ -26,6 +26,7 @@ public ProfileEditorFundingDecision()
             AmountInEur = -1;
             AmountInFundingDecisionCurrency = null;
             FundingDecisionCurrencyAbbreviation = null;
+            Url = "";
         }
 
         public int ProjectId { get; set; }
@@ -51,5 +52,6 @@ public ProfileEditorFundingDecision()
         public decimal AmountInEur { get; set; }
         public decimal? AmountInFundingDecisionCurrency { get; set; }
         public string? FundingDecisionCurrencyAbbreviation { get; set; }
+        public string Url { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Models/ProfileEditor/Items/ProfileEditorResearchDataset.cs b/aspnetcore/src/api/Models/ProfileEditor/Items/ProfileEditorResearchDataset.cs
@@ -14,6 +14,7 @@ public ProfileEditorResearchDataset()
             DescriptionFi = "";
             DescriptionSv = "";
             DescriptionEn = "";
+            Url = "";
             DatasetCreated = null;
             PreferredIdentifiers = new List<ProfileEditorPreferredIdentifier>();
         }
@@ -27,6 +28,7 @@ public ProfileEditorResearchDataset()
         public string DescriptionFi { get; set; }
         public string DescriptionSv { get; set; }
         public string DescriptionEn { get; set; }
+        public string Url { get; set; }
         public int? DatasetCreated { get; set; }
         public List<ProfileEditorPreferredIdentifier> PreferredIdentifiers { get; set; }
     }

diff --git a/aspnetcore/src/api/Models/ProfileEditor/ProfileDataFromSql.cs b/aspnetcore/src/api/Models/ProfileEditor/ProfileDataFromSql.cs
@@ -179,6 +179,7 @@ public ProfileDataFromSql()
         public string DimProfileOnlyResearchActivity_Role_NameFi { get; set; }
         public string DimProfileOnlyResearchActivity_Role_NameEn { get; set; }
         public string DimProfileOnlyResearchActivity_Role_NameSv { get; set; }
+        public string DimProfileOnlyResearchActivity_DimWebLink_Url { get; set; }
         // DimResearchActivity
         public int DimResearchActivity_DimOrganization_Id { get; set; }
         public string DimResearchActivity_DimOrganization_OrganizationId { get; set; }
@@ -262,6 +263,7 @@ public ProfileDataFromSql()
         public string DimProfileOnlyFundingDecision_DimTypeOfFunding_NameFi { get; set; }
         public string DimProfileOnlyFundingDecision_DimTypeOfFunding_NameEn { get; set; }
         public string DimProfileOnlyFundingDecision_DimTypeOfFunding_NameSv { get; set; }
+        public string DimProfileOnlyFundingDecision_DimWebLink_Url { get; set; }
         // DimResearchDataset
         public string DimResearchDataset_LocalIdentifier { get; set; }
         public string DimResearchDataset_NameFi { get; set; }
@@ -280,5 +282,6 @@ public ProfileDataFromSql()
         public string DimProfileOnlyDataset_DescriptionEn { get; set; }
         public string DimProfileOnlyDataset_DescriptionSv { get; set; }
         public DateTime? DimProfileOnlyDataset_DatasetCreated { get; set; }
+        public string DimProfileOnlyDataset_DimWebLink_Url { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Models/StructuredLog/LogContent.cs b/aspnetcore/src/api/Models/StructuredLog/LogContent.cs
@@ -13,6 +13,8 @@ public static class Action
             public const string ADMIN_ELASTICSEARCH_PROFILE_DELETE = "Admin: Elasticsearch: profile: delete";
             public const string ADMIN_ELASTICSEARCH_PROFILE_UPDATE = "Admin: Elasticsearch: profile: update";
             public const string ADMIN_ELASTICSEARCH_PROFILE_UPDATE_ALL = "Admin: Elasticsearch: profile: update all";
+            public const string ADMIN_ORCID_UPDATE_ALL = "Admin: ORCID: profile: update all";
+            public const string ADMIN_HEALTH_CHECK = "Admin: Health check";
             public const string BACKGROUND_UPDATE = "Background: update";
             public const string ELASTICSEARCH_DELETE = "Elasticsearch: profile: delete";
             public const string ELASTICSEARCH_UPDATE = "Elasticsearch: profile: update";
@@ -42,6 +44,7 @@ public static class Action
         public static class ActionState
         {
             public const string START = "start";
+            public const string IN_PROGRESS = "in progress";
             public const string COMPLETE = "complete";
             public const string FAILED = "failed";
             public const string CANCELLED = "cancelled";

diff --git a/aspnetcore/src/api/Models/Ttv/DimProfileOnlyDataset.cs b/aspnetcore/src/api/Models/Ttv/DimProfileOnlyDataset.cs
@@ -8,6 +8,7 @@ public partial class DimProfileOnlyDataset
         public DimProfileOnlyDataset()
         {
             DimPids = new HashSet<DimPid>();
+            DimWebLinks = new HashSet<DimWebLink>();
             FactFieldValues = new HashSet<FactFieldValue>();
         }
 
@@ -34,6 +35,7 @@ public DimProfileOnlyDataset()
         public virtual DimReferencedatum DimReferencedataIdAvailabilityNavigation { get; set; }
         public virtual DimRegisteredDataSource DimRegisteredDataSource { get; set; }
         public virtual ICollection<DimPid> DimPids { get; set; }
+        public virtual ICollection<DimWebLink> DimWebLinks { get; set; }
         public virtual ICollection<FactFieldValue> FactFieldValues { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Models/Ttv/DimProfileOnlyFundingDecision.cs b/aspnetcore/src/api/Models/Ttv/DimProfileOnlyFundingDecision.cs
@@ -8,6 +8,7 @@ public partial class DimProfileOnlyFundingDecision
         public DimProfileOnlyFundingDecision()
         {
             DimPids = new HashSet<DimPid>();
+            DimWebLinks = new HashSet<DimWebLink>();
             FactFieldValues = new HashSet<FactFieldValue>();
         }
 
@@ -45,6 +46,7 @@ public DimProfileOnlyFundingDecision()
         public virtual DimRegisteredDataSource DimRegisteredDataSource { get; set; }
         public virtual DimTypeOfFunding DimTypeOfFunding { get; set; }
         public virtual ICollection<DimPid> DimPids { get; set; }
+        public virtual ICollection<DimWebLink> DimWebLinks { get; set; }
         public virtual ICollection<FactFieldValue> FactFieldValues { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Models/Ttv/DimProfileOnlyResearchActivity.cs b/aspnetcore/src/api/Models/Ttv/DimProfileOnlyResearchActivity.cs
@@ -7,6 +7,7 @@ public partial class DimProfileOnlyResearchActivity
     {
         public DimProfileOnlyResearchActivity()
         {
+            DimWebLinks = new HashSet<DimWebLink>();
             FactFieldValues = new HashSet<FactFieldValue>();
         }
 
@@ -38,6 +39,7 @@ public DimProfileOnlyResearchActivity()
         public virtual DimGeo DimGeoIdCountryNavigation { get; set; }
         public virtual DimOrganization DimOrganization { get; set; }
         public virtual DimRegisteredDataSource DimRegisteredDataSource { get; set; }
+        public virtual ICollection<DimWebLink> DimWebLinks { get; set; }
         public virtual ICollection<FactFieldValue> FactFieldValues { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Models/Ttv/DimPublication.cs b/aspnetcore/src/api/Models/Ttv/DimPublication.cs
@@ -44,15 +44,15 @@ public DimPublication()
         public string ParentPublicationName { get; set; }
         public string ParentPublicationPublisher { get; set; }
         public string PublicationTypeCode { get; set; }
-        public bool InternationalCollaboration { get; set; }
+        public bool? InternationalCollaboration { get; set; }
         public bool HospitalDistrictCollaboration { get; set; }
         public int InternationalPublication { get; set; }
         public bool GovermentCollaboration { get; set; }
         public bool OtherCollaboration { get; set; }
         public string LanguageCode { get; set; }
         public string OpenAccessCode { get; set; }
         public bool SpecialStateSubsidy { get; set; }
-        public bool BusinessCollaboration { get; set; }
+        public bool? BusinessCollaboration { get; set; }
         public string DoiHandle { get; set; }
         public string JuuliAddress { get; set; }
         public string OriginalPublicationId { get; set; }