Release 1.29 (#220)

* Improve error handling in ORCID record import (#212)

* Prevent duplicate key error in fact_field_values when adding TTV data via fact_contribution (#213)

* Cscttv 3532 keycloak upgrade from 19 to 20 (#214)

* Upgrade Keycloak from 19.0.3 to 20.0.5

* Define target platform in Keycloak Docker images

* TTV model update - jufo classes (#215)

* Additional TTV model update in tables fact_jufo_class_codes_for_pub_channels and dim_publication_channel (#216)

* Fix bug in DimProfileOnlyFundingDecision date handling (#217)

* CSCTTV-3561 database model update to dim_publication.jufo_class (#219)

aspnetcore/src/api/Controllers/OrcidController.cs

@@ -1,6 +1,6 @@
-using api.Services;
-using api.Models.Api;
-using api.Models.Orcid;
+using api.Services;
+using api.Models.Api;
+using api.Models.Orcid;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using System.Threading.Tasks;
@@ -209,7 +209,7 @@ public async Task<IActionResult> Get()
                                         action: LogContent.Action.ORCID_RECORD_IMPORT,
                                         state: LogContent.ActionState.START));
 
-                await _orcidImportService.ImportOrcidRecordJsonIntoUserProfile(userprofileId, orcidRecordJson);
+                await _orcidImportService.ImportOrcidRecordJsonIntoUserProfile(userprofileId, orcidRecordJson, logUserIdentification);
                 importSuccess = true;
 
                 _logger.LogInformation(
@@ -264,7 +264,7 @@ await _taskQueue.QueueBackgroundWorkItemAsync(async token =>
                             .Include(ffv => ffv.DimProfileOnlyFundingDecision)
                             .Include(ffv => ffv.DimPidIdOrcidPutCodeNavigation).ToListAsync();
 
-                        await _orcidImportService.ImportAdditionalData(factFieldValues: ffvs, orcidAccessToken: orcidTokens.AccessToken, useOrcidPublicApi: useOrcidPublicApi);
+                        await _orcidImportService.ImportAdditionalData(factFieldValues: ffvs, orcidAccessToken: orcidTokens.AccessToken, logUserIdentification: logUserIdentification, useOrcidPublicApi: useOrcidPublicApi);
                         await localTtvContext.SaveChangesAsync();
 
                         _logger.LogInformation(

aspnetcore/src/api/Controllers/WebhookController.cs

@@ -159,7 +159,7 @@ await _taskQueue.QueueBackgroundWorkItemAsync(async token =>
                                 action: LogContent.Action.ORCID_RECORD_IMPORT,
                                 state: LogContent.ActionState.START));
 
-                        importSuccess = await localOrcidImportService.ImportOrcidRecordJsonIntoUserProfile(dimUserprofileId, orcidRecordJson);
+                        importSuccess = await localOrcidImportService.ImportOrcidRecordJsonIntoUserProfile(dimUserprofileId, orcidRecordJson, logUserIdentification);
 
                         _logger.LogInformation(
                             LogContent.MESSAGE_TEMPLATE,
@@ -202,7 +202,7 @@ await _taskQueue.QueueBackgroundWorkItemAsync(async token =>
                             .Include(ffv => ffv.DimProfileOnlyFundingDecision)
                             .Include(ffv => ffv.DimPidIdOrcidPutCodeNavigation).ToListAsync();
 
-                        await localOrcidImportService.ImportAdditionalData(factFieldValues: ffvs, orcidAccessToken: orcidAccessToken);
+                        await localOrcidImportService.ImportAdditionalData(factFieldValues: ffvs, orcidAccessToken: orcidAccessToken, logUserIdentification: logUserIdentification);
                         await localTtvContext.SaveChangesAsync();
 
                         _logger.LogInformation(

aspnetcore/src/api/Models/StructuredLog/LogContent.cs

@@ -35,6 +35,7 @@ public static class Action
             public const string ORCID_RECORD_GET_PUBLIC_API = "ORCID: record: get from public API";
             public const string ORCID_RECORD_IMPORT = "ORCID: record: import";
             public const string ORCID_RECORD_IMPORT_ADDITIONAL = "ORCID: record: import additional";
+            public const string ORCID_RECORD_PARSE = "ORCID: record: parse";
             public const string ORCID_REVOKE_TOKEN = "ORCID: revoke token";
             public const string ORCID_WEBHOOK_REGISTER = "ORCID: webhook: register";
             public const string ORCID_WEBHOOK_UNREGISTER = "ORCID: webhook: unregister";

aspnetcore/src/api/Models/Ttv/DimPublication.cs

@@ -29,10 +29,6 @@ public partial class DimPublication
 
     public string Isbn2 { get; set; }
 
-    public string JufoCode { get; set; }
-
-    public string JufoClassCode { get; set; }
-
     public int PublicationCountryCode { get; set; }
 
     public string JournalName { get; set; }
@@ -69,8 +65,6 @@ public partial class DimPublication
 
     public int LanguageCode { get; set; }
 
-    public string OpenAccessCode { get; set; }
-
     public bool SpecialStateSubsidy { get; set; }
 
     public bool? BusinessCollaboration { get; set; }
@@ -119,16 +113,22 @@ public partial class DimPublication
 
     public string OpenAccess { get; set; }
 
-    public string PublisherOpenAccessCode { get; set; }
+    public int PublisherOpenAccessCode { get; set; }
 
     public string Abstract { get; set; }
 
+    public int DimPublicationChannelId { get; set; }
+
+    public int JufoClass { get; set; }
+
     public virtual DimReferencedatum ArticleTypeCodeNavigation { get; set; }
 
     public virtual ICollection<DimLocallyReportedPubInfo> DimLocallyReportedPubInfos { get; set; } = new List<DimLocallyReportedPubInfo>();
 
     public virtual ICollection<DimPid> DimPids { get; set; } = new List<DimPid>();
 
+    public virtual DimPublicationChannel DimPublicationChannel { get; set; }
+
     public virtual DimRegisteredDataSource DimRegisteredDataSource { get; set; }
 
     public virtual ICollection<FactContribution> FactContributions { get; set; } = new List<FactContribution>();
@@ -137,6 +137,8 @@ public partial class DimPublication
 
     public virtual ICollection<FactFieldValue> FactFieldValues { get; set; } = new List<FactFieldValue>();
 
+    public virtual DimReferencedatum JufoClassNavigation { get; set; }
+
     public virtual DimReferencedatum LanguageCodeNavigation { get; set; }
 
     public virtual DimReferencedatum LicenseCodeNavigation { get; set; }
@@ -149,6 +151,8 @@ public partial class DimPublication
 
     public virtual DimReferencedatum PublicationTypeCodeNavigation { get; set; }
 
+    public virtual DimReferencedatum PublisherOpenAccessCodeNavigation { get; set; }
+
     public virtual DimReferencedatum TargetAudienceCodeNavigation { get; set; }
 
     public virtual DimReferencedatum ThesisTypeCodeNavigation { get; set; }

aspnetcore/src/api/Models/Ttv/DimPublicationChannel.cs

@@ -13,9 +13,17 @@ public partial class DimPublicationChannel
 
     public string PublisherNameText { get; set; }
 
+    public string SourceId { get; set; }
+
+    public string SourceDescription { get; set; }
+
+    public DateTime? Created { get; set; }
+
+    public DateTime? Modified { get; set; }
+
     public virtual ICollection<DimPid> DimPids { get; set; } = new List<DimPid>();
 
-    public virtual ICollection<DimResearchActivity> DimResearchActivities { get; set; } = new List<DimResearchActivity>();
+    public virtual ICollection<DimPublication> DimPublications { get; set; } = new List<DimPublication>();
 
-    public virtual ICollection<FactJufoClassCodesForPubChannel> FactJufoClassCodesForPubChannels { get; set; } = new List<FactJufoClassCodesForPubChannel>();
+    public virtual ICollection<DimResearchActivity> DimResearchActivities { get; set; } = new List<DimResearchActivity>();
 }

aspnetcore/src/api/Models/Ttv/DimReferencedatum.cs

@@ -63,6 +63,8 @@ public partial class DimReferencedatum
 
     public virtual ICollection<DimPublication> DimPublicationArticleTypeCodeNavigations { get; set; } = new List<DimPublication>();
 
+    public virtual ICollection<DimPublication> DimPublicationJufoClassNavigations { get; set; } = new List<DimPublication>();
+
     public virtual ICollection<DimPublication> DimPublicationLanguageCodeNavigations { get; set; } = new List<DimPublication>();
 
     public virtual ICollection<DimPublication> DimPublicationLicenseCodeNavigations { get; set; } = new List<DimPublication>();
@@ -75,6 +77,8 @@ public partial class DimReferencedatum
 
     public virtual ICollection<DimPublication> DimPublicationPublicationTypeCodeNavigations { get; set; } = new List<DimPublication>();
 
+    public virtual ICollection<DimPublication> DimPublicationPublisherOpenAccessCodeNavigations { get; set; } = new List<DimPublication>();
+
     public virtual ICollection<DimPublication> DimPublicationTargetAudienceCodeNavigations { get; set; } = new List<DimPublication>();
 
     public virtual ICollection<DimPublication> DimPublicationThesisTypeCodeNavigations { get; set; } = new List<DimPublication>();
@@ -95,8 +99,6 @@ public partial class DimReferencedatum
 
     public virtual ICollection<FactFieldValue> FactFieldValueDimReferencedataFieldOfSciences { get; set; } = new List<FactFieldValue>();
 
-    public virtual ICollection<FactJufoClassCodesForPubChannel> FactJufoClassCodesForPubChannels { get; set; } = new List<FactJufoClassCodesForPubChannel>();
-
     public virtual ICollection<DimReferencedatum> InverseDimReferencedata { get; set; } = new List<DimReferencedatum>();
 
     public virtual ICollection<DimCallProgramme> DimCallProgrammes { get; set; } = new List<DimCallProgramme>();

aspnetcore/src/api/Models/Ttv/FactJufoClassCodesForPubChannel.cs

@@ -7,11 +7,19 @@ public partial class FactJufoClassCodesForPubChannel
 {
     public int DimPublicationChannelId { get; set; }
 
-    public int DimReferencedataId { get; set; }
+    public int JufoClasses { get; set; }
 
     public int Year { get; set; }
 
+    public string SourceId { get; set; }
+
+    public string SourceDescription { get; set; }
+
+    public DateTime? Created { get; set; }
+
+    public DateTime? Modified { get; set; }
+
     public virtual DimPublicationChannel DimPublicationChannel { get; set; }
 
-    public virtual DimReferencedatum DimReferencedata { get; set; }
+    public virtual DimReferencedatum JufoClassesNavigation { get; set; }
 }

aspnetcore/src/api/Models/Ttv/TtvContext.cs

@@ -137,8 +137,6 @@ public TtvContext(DbContextOptions<TtvContext> options)
 
     public virtual DbSet<FactInfraKeyword> FactInfraKeywords { get; set; }
 
-    public virtual DbSet<FactJufoClassCodesForPubChannel> FactJufoClassCodesForPubChannels { get; set; }
-
     public virtual DbSet<FactUpkeep> FactUpkeeps { get; set; }
 
     protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
@@ -2430,6 +2428,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
             entity.Property(e => e.Created)
                 .HasColumnType("datetime")
                 .HasColumnName("created");
+            entity.Property(e => e.DimPublicationChannelId).HasColumnName("dim_publication_channel_id");
             entity.Property(e => e.DimRegisteredDataSourceId).HasColumnName("dim_registered_data_source_id");
             entity.Property(e => e.Doi)
                 .HasMaxLength(4000)
@@ -2459,12 +2458,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
             entity.Property(e => e.JournalName)
                 .HasMaxLength(4000)
                 .HasColumnName("journal_name");
-            entity.Property(e => e.JufoClassCode)
-                .HasMaxLength(255)
-                .HasColumnName("jufo_class_code");
-            entity.Property(e => e.JufoCode)
-                .HasMaxLength(255)
-                .HasColumnName("jufo_code");
+            entity.Property(e => e.JufoClass).HasColumnName("jufo_class");
             entity.Property(e => e.JuuliAddress)
                 .HasMaxLength(4000)
                 .HasColumnName("juuli_address");
@@ -2477,9 +2471,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
             entity.Property(e => e.OpenAccess)
                 .HasMaxLength(255)
                 .HasColumnName("open_access");
-            entity.Property(e => e.OpenAccessCode)
-                .HasMaxLength(255)
-                .HasColumnName("open_access_code");
             entity.Property(e => e.OriginalPublicationId)
                 .HasMaxLength(255)
                 .HasColumnName("original_publication_id");
@@ -2520,9 +2511,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
             entity.Property(e => e.PublisherName)
                 .HasMaxLength(4000)
                 .HasColumnName("publisher_name");
-            entity.Property(e => e.PublisherOpenAccessCode)
-                .HasMaxLength(255)
-                .HasColumnName("publisher_open_access_code");
+            entity.Property(e => e.PublisherOpenAccessCode).HasColumnName("publisher_open_access_code");
             entity.Property(e => e.Report).HasColumnName("report");
             entity.Property(e => e.ReportingYear).HasColumnName("reporting_year");
             entity.Property(e => e.SelfArchivedCode).HasColumnName("self_archived_code");
@@ -2544,11 +2533,21 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 .HasForeignKey(d => d.ArticleTypeCode)
                 .HasConstraintName("article_type_code");
 
+            entity.HasOne(d => d.DimPublicationChannel).WithMany(p => p.DimPublications)
+                .HasForeignKey(d => d.DimPublicationChannelId)
+                .OnDelete(DeleteBehavior.ClientSetNull)
+                .HasConstraintName("publication_channel");
+
             entity.HasOne(d => d.DimRegisteredDataSource).WithMany(p => p.DimPublications)
                 .HasForeignKey(d => d.DimRegisteredDataSourceId)
                 .OnDelete(DeleteBehavior.ClientSetNull)
                 .HasConstraintName("FKdim_public896887");
 
+            entity.HasOne(d => d.JufoClassNavigation).WithMany(p => p.DimPublicationJufoClassNavigations)
+                .HasForeignKey(d => d.JufoClass)
+                .OnDelete(DeleteBehavior.ClientSetNull)
+                .HasConstraintName("jufo_class");
+
             entity.HasOne(d => d.LanguageCodeNavigation).WithMany(p => p.DimPublicationLanguageCodeNavigations)
                 .HasForeignKey(d => d.LanguageCode)
                 .OnDelete(DeleteBehavior.ClientSetNull)
@@ -2577,6 +2576,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 .HasForeignKey(d => d.PublicationTypeCode2)
                 .HasConstraintName("publication_type_code2");
 
+            entity.HasOne(d => d.PublisherOpenAccessCodeNavigation).WithMany(p => p.DimPublicationPublisherOpenAccessCodeNavigations)
+                .HasForeignKey(d => d.PublisherOpenAccessCode)
+                .OnDelete(DeleteBehavior.ClientSetNull)
+                .HasConstraintName("publisher_open_access");
+
             entity.HasOne(d => d.TargetAudienceCodeNavigation).WithMany(p => p.DimPublicationTargetAudienceCodeNavigations)
                 .HasForeignKey(d => d.TargetAudienceCode)
                 .HasConstraintName("target_audience_code");
@@ -2616,12 +2620,25 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
             entity.Property(e => e.ChannelNameAnylang)
                 .HasMaxLength(4000)
                 .HasColumnName("channel_name_anylang");
+            entity.Property(e => e.Created)
+                .HasColumnType("datetime")
+                .HasColumnName("created");
             entity.Property(e => e.JufoCode)
                 .HasMaxLength(255)
                 .HasColumnName("jufo_code");
+            entity.Property(e => e.Modified)
+                .HasColumnType("datetime")
+                .HasColumnName("modified");
             entity.Property(e => e.PublisherNameText)
                 .HasMaxLength(4000)
                 .HasColumnName("publisher_name_text");
+            entity.Property(e => e.SourceDescription)
+                .HasMaxLength(255)
+                .HasColumnName("source_description");
+            entity.Property(e => e.SourceId)
+                .IsRequired()
+                .HasMaxLength(255)
+                .HasColumnName("source_id");
         });
 
         modelBuilder.Entity<DimPurpose>(entity =>
@@ -4020,27 +4037,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 .HasConstraintName("FKfact_infra619117");
         });
 
-        modelBuilder.Entity<FactJufoClassCodesForPubChannel>(entity =>
-        {
-            entity.HasKey(e => new { e.DimPublicationChannelId, e.DimReferencedataId, e.Year }).HasName("PK__fact_juf__0E099E4B39813744");
-
-            entity.ToTable("fact_jufo_class_codes_for_pub_channels");
-
-            entity.Property(e => e.DimPublicationChannelId).HasColumnName("dim_publication_channel_id");
-            entity.Property(e => e.DimReferencedataId).HasColumnName("dim_referencedata_id");
-            entity.Property(e => e.Year).HasColumnName("year");
-
-            entity.HasOne(d => d.DimPublicationChannel).WithMany(p => p.FactJufoClassCodesForPubChannels)
-                .HasForeignKey(d => d.DimPublicationChannelId)
-                .OnDelete(DeleteBehavior.ClientSetNull)
-                .HasConstraintName("jufo_classes");
-
-            entity.HasOne(d => d.DimReferencedata).WithMany(p => p.FactJufoClassCodesForPubChannels)
-                .HasForeignKey(d => d.DimReferencedataId)
-                .OnDelete(DeleteBehavior.ClientSetNull)
-                .HasConstraintName("FKfact_jufo_876058");
-        });
-
         modelBuilder.Entity<FactUpkeep>(entity =>
         {
             entity.HasKey(e => new { e.DimOrganizationId, e.DimGeoId, e.DimInfrastructureId, e.DimServiceId, e.DimServicePointId, e.DimDateIdStart, e.DimDateIdEnd }).HasName("PK__fact_upk__850A8E30119C2EB4");

aspnetcore/src/api/Services/IOrcidImportService.cs

@@ -1,13 +1,14 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using api.Models.Log;
 using api.Models.Ttv;
 
 namespace api.Services
 {
     public interface IOrcidImportService
     {
-        Task<bool> ImportOrcidRecordJsonIntoUserProfile(int userprofileId, string json);
-        Task<bool> ImportAdditionalData(List<FactFieldValue> factFieldValues, String orcidAccessToken, bool useOrcidPublicApi = false);
+        Task<bool> ImportOrcidRecordJsonIntoUserProfile(int userprofileId, string json, LogUserIdentification logUserIdentification);
+        Task<bool> ImportAdditionalData(List<FactFieldValue> factFieldValues, String orcidAccessToken, LogUserIdentification logUserIdentification, bool useOrcidPublicApi = false);
     }
 }